Merge branch 'deploy-fixes' into 'master'
`fdroid deploy` fixes See merge request fdroid/fdroidserver!703
This commit is contained in:
Michael Pöhn
commit
9a8718c189
|
|
@ -190,6 +190,13 @@ The repository of older versions of applications from the main demo repository.
|
|||
# 'https://gitlab.com/user/repo',
|
||||
# }
|
||||
|
||||
# Most git hosting services have hard size limits for each git repo.
|
||||
# `fdroid deploy` will delete the git history when the git mirror repo
|
||||
# approaches this limit to ensure that the repo will still fit when
|
||||
# pushed. GitHub recommends 1GB, gitlab.com recommends 10GB.
|
||||
#
|
||||
# git_mirror_size_limit = '10GB'
|
||||
|
||||
# Any mirrors of this repo, for example all of the servers declared in
|
||||
# serverwebroot and all the servers declared in servergitmirrors,
|
||||
# will automatically be used by the client. If one
|
||||
|
|
|
|||
|
|
@ -148,6 +148,7 @@ default_config = {
|
|||
'archive_description': _('These are the apps that have been archived from the main repo.'),
|
||||
'archive_older': 0,
|
||||
'lint_licenses': fdroidserver.lint.APPROVED_LICENSES,
|
||||
'git_mirror_size_limit': 10000000000,
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -354,9 +355,31 @@ def read_config(opts, config_file='config.py'):
|
|||
raise TypeError(_('only accepts strings, lists, and tuples'))
|
||||
config['servergitmirrors'] = roots
|
||||
|
||||
limit = config['git_mirror_size_limit']
|
||||
config['git_mirror_size_limit'] = parse_human_readable_size(limit)
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def parse_human_readable_size(size):
|
||||
units = {
|
||||
'b': 1,
|
||||
'kb': 1000, 'mb': 1000**2, 'gb': 1000**3, 'tb': 1000**4,
|
||||
'kib': 1024, 'mib': 1024**2, 'gib': 1024**3, 'tib': 1024**4,
|
||||
}
|
||||
try:
|
||||
return int(float(size))
|
||||
except (ValueError, TypeError):
|
||||
if type(size) != str:
|
||||
raise ValueError(_('Could not parse size "{size}", wrong type "{type}"')
|
||||
.format(size=size, type=type(size)))
|
||||
s = size.lower().replace(' ', '')
|
||||
m = re.match(r'^(?P<value>[0-9][0-9.]+) *(?P<unit>' + r'|'.join(units.keys()) + r')$', s)
|
||||
if not m:
|
||||
raise ValueError(_('Not a valid size definition: "{}"').format(size))
|
||||
return int(float(m.group("value")) * units[m.group("unit")])
|
||||
|
||||
|
||||
def assert_config_keystore(config):
|
||||
"""Check weather keystore is configured correctly and raise exception if not."""
|
||||
|
||||
|
|
|
|||
|
|
@ -48,6 +48,10 @@ def main():
|
|||
+ 'using the query string: ?fingerprint='))
|
||||
parser.add_argument("--archive", action='store_true', default=False,
|
||||
help=_("Also mirror the full archive section"))
|
||||
parser.add_argument("--build-logs", action='store_true', default=False,
|
||||
help=_("Include the build logs in the mirror"))
|
||||
parser.add_argument("--src-tarballs", action='store_true', default=False,
|
||||
help=_("Include the source tarballs in the mirror"))
|
||||
parser.add_argument("--output-dir", default=None,
|
||||
help=_("The directory to write the mirror to"))
|
||||
options = parser.parse_args()
|
||||
|
|
@ -135,7 +139,10 @@ def main():
|
|||
for packageName, packageList in data['packages'].items():
|
||||
for package in packageList:
|
||||
to_fetch = []
|
||||
for k in ('apkName', 'srcname'):
|
||||
keys = ['apkName', ]
|
||||
if options.src_tarballs:
|
||||
keys.append('srcname')
|
||||
for k in keys:
|
||||
if k in package:
|
||||
to_fetch.append(package[k])
|
||||
elif k == 'apkName':
|
||||
|
|
@ -146,6 +153,9 @@ def main():
|
|||
or (f.endswith('.apk') and os.path.getsize(f) != package['size']):
|
||||
urls.append(_append_to_url_path(section, f))
|
||||
urls.append(_append_to_url_path(section, f + '.asc'))
|
||||
if options.build_logs and f.endswith('.apk'):
|
||||
urls.append(_append_to_url_path(section, f[:-4] + '.log.gz'))
|
||||
|
||||
_run_wget(sectiondir, urls)
|
||||
|
||||
for app in data['apps']:
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import pwd
|
|||
import re
|
||||
import subprocess
|
||||
import time
|
||||
import urllib
|
||||
from argparse import ArgumentParser
|
||||
import logging
|
||||
import shutil
|
||||
|
|
@ -352,11 +353,18 @@ def update_servergitmirrors(servergitmirrors, repo_section):
|
|||
git_repodir = os.path.join(git_mirror_path, 'fdroid', repo_section)
|
||||
if not os.path.isdir(git_repodir):
|
||||
os.makedirs(git_repodir)
|
||||
if os.path.isdir(dotgit) and _get_size(git_mirror_path) > 1000000000:
|
||||
logging.warning('Deleting git-mirror history, repo is too big (1 gig max)')
|
||||
# github/gitlab use bare git repos, so only count the .git folder
|
||||
# test: generate giant APKs by including AndroidManifest.xml and and large
|
||||
# file from /dev/urandom, then sign it. Then add those to the git repo.
|
||||
dotgit_size = _get_size(dotgit)
|
||||
dotgit_over_limit = dotgit_size > config['git_mirror_size_limit']
|
||||
if os.path.isdir(dotgit) and dotgit_over_limit:
|
||||
logging.warning(_('Deleting git-mirror history, repo is too big ({size} max {limit})')
|
||||
.format(size=dotgit_size, limit=config['git_mirror_size_limit']))
|
||||
shutil.rmtree(dotgit)
|
||||
if options.no_keep_git_mirror_archive and _get_size(git_mirror_path) > 1000000000:
|
||||
logging.warning('Deleting archive, repo is too big (1 gig max)')
|
||||
if options.no_keep_git_mirror_archive and dotgit_over_limit:
|
||||
logging.warning(_('Deleting archive, repo is too big ({size} max {limit})')
|
||||
.format(size=dotgit_size, limit=config['git_mirror_size_limit']))
|
||||
archive_path = os.path.join(git_mirror_path, 'fdroid', 'archive')
|
||||
shutil.rmtree(archive_path, ignore_errors=True)
|
||||
|
||||
|
|
@ -473,7 +481,7 @@ def upload_to_android_observatory(repo_section):
|
|||
logging.info(message)
|
||||
|
||||
|
||||
def upload_to_virustotal(repo_section, vt_apikey):
|
||||
def upload_to_virustotal(repo_section, virustotal_apikey):
|
||||
import json
|
||||
import requests
|
||||
|
||||
|
|
@ -506,13 +514,13 @@ def upload_to_virustotal(repo_section, vt_apikey):
|
|||
"User-Agent": "F-Droid"
|
||||
}
|
||||
data = {
|
||||
'apikey': vt_apikey,
|
||||
'apikey': virustotal_apikey,
|
||||
'resource': package['hash'],
|
||||
}
|
||||
needs_file_upload = False
|
||||
while True:
|
||||
r = requests.post('https://www.virustotal.com/vtapi/v2/file/report',
|
||||
data=data, headers=headers)
|
||||
r = requests.get('https://www.virustotal.com/vtapi/v2/file/report?'
|
||||
+ urllib.parse.urlencode(data), headers=headers)
|
||||
if r.status_code == 200:
|
||||
response = r.json()
|
||||
if response['response_code'] == 0:
|
||||
|
|
@ -533,18 +541,40 @@ def upload_to_virustotal(repo_section, vt_apikey):
|
|||
elif r.status_code == 204:
|
||||
time.sleep(10) # wait for public API rate limiting
|
||||
|
||||
upload_url = None
|
||||
if needs_file_upload:
|
||||
logging.info('Uploading ' + repofilename + ' to virustotal')
|
||||
manual_url = 'https://www.virustotal.com/'
|
||||
size = os.path.getsize(repofilename)
|
||||
if size > 200000000:
|
||||
# VirusTotal API 200MB hard limit
|
||||
logging.error(_('{path} more than 200MB, manually upload: {url}')
|
||||
.format(path=repofilename, url=manual_url))
|
||||
elif size > 32000000:
|
||||
# VirusTotal API requires fetching a URL to upload bigger files
|
||||
r = requests.get('https://www.virustotal.com/vtapi/v2/file/scan/upload_url?'
|
||||
+ urllib.parse.urlencode(data), headers=headers)
|
||||
if r.status_code == 200:
|
||||
upload_url = r.json().get('upload_url')
|
||||
elif r.status_code == 403:
|
||||
logging.error(_('VirusTotal API key cannot upload files larger than 32MB, '
|
||||
+ 'use {url} to upload {path}.')
|
||||
.format(path=repofilename, url=manual_url))
|
||||
else:
|
||||
r.raise_for_status()
|
||||
else:
|
||||
upload_url = 'https://www.virustotal.com/vtapi/v2/file/scan'
|
||||
|
||||
if upload_url:
|
||||
logging.info(_('Uploading {apkfilename} to virustotal')
|
||||
.format(apkfilename=repofilename))
|
||||
files = {
|
||||
'file': (filename, open(repofilename, 'rb'))
|
||||
}
|
||||
r = requests.post('https://www.virustotal.com/vtapi/v2/file/scan',
|
||||
data=data, headers=headers, files=files)
|
||||
logging.debug('If this upload fails, try manually uploading here:\n'
|
||||
+ 'https://www.virustotal.com/')
|
||||
r = requests.post(upload_url, data=data, headers=headers, files=files)
|
||||
logging.debug(_('If this upload fails, try manually uploading to {url}')
|
||||
.format(url=manual_url))
|
||||
r.raise_for_status()
|
||||
response = r.json()
|
||||
|
||||
logging.info(response['verbose_msg'] + " " + response['permalink'])
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -45,6 +45,15 @@ class CommonTest(unittest.TestCase):
|
|||
os.makedirs(self.tmpdir)
|
||||
os.chdir(self.basedir)
|
||||
|
||||
def test_parse_human_readable_size(self):
|
||||
for k, v in ((9827, 9827), (123.456, 123), ('123b', 123), ('1.2', 1),
|
||||
('10.43 KiB', 10680), ('11GB', 11000000000), ('59kb', 59000),
|
||||
('343.1 mb', 343100000), ('99.9GiB', 107266808217)):
|
||||
self.assertEqual(fdroidserver.common.parse_human_readable_size(k), v)
|
||||
for v in ((12, 123), '0xfff', [], None, '12,123', '123GG', '982374bb', self):
|
||||
with self.assertRaises(ValueError):
|
||||
fdroidserver.common.parse_human_readable_size(v)
|
||||
|
||||
def test_assert_config_keystore(self):
|
||||
with tempfile.TemporaryDirectory() as tmpdir, TmpCwd(tmpdir):
|
||||
with self.assertRaises(FDroidException):
|
||||
|
|
|
|||
|
|
@ -1096,6 +1096,59 @@ $fdroid update --create-key
|
|||
test -e $KEYSTORE
|
||||
|
||||
|
||||
#------------------------------------------------------------------------------#
|
||||
echo_header "setup a new repo from scratch using ANDROID_HOME with git mirror"
|
||||
|
||||
# fake git remote server for repo mirror
|
||||
SERVER_GIT_MIRROR=`create_test_dir`
|
||||
cd $SERVER_GIT_MIRROR
|
||||
git init
|
||||
git config receive.denyCurrentBranch updateInstead
|
||||
|
||||
REPOROOT=`create_test_dir`
|
||||
GIT_MIRROR=$REPOROOT/git-mirror
|
||||
cd $REPOROOT
|
||||
fdroid_init_with_prebuilt_keystore
|
||||
echo "servergitmirrors = '$SERVER_GIT_MIRROR'" >> config.py
|
||||
|
||||
cp $WORKSPACE/tests/repo/com.politedroid_[345].apk repo/
|
||||
$fdroid update --create-metadata
|
||||
$fdroid deploy
|
||||
test -e $GIT_MIRROR/fdroid/repo/com.politedroid_3.apk
|
||||
test -e $GIT_MIRROR/fdroid/repo/com.politedroid_4.apk
|
||||
test -e $GIT_MIRROR/fdroid/repo/com.politedroid_5.apk
|
||||
test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_3.apk
|
||||
test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_4.apk
|
||||
test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_5.apk
|
||||
date > $GIT_MIRROR/.git/test-stamp
|
||||
|
||||
# add one more APK to trigger archiving
|
||||
cp $WORKSPACE/tests/repo/com.politedroid_6.apk repo/
|
||||
$fdroid update
|
||||
$fdroid deploy
|
||||
test -e $REPOROOT/archive/com.politedroid_3.apk
|
||||
! test -e $GIT_MIRROR/fdroid/archive/com.politedroid_3.apk
|
||||
! test -e $SERVER_GIT_MIRROR/fdroid/archive/com.politedroid_3.apk
|
||||
test -e $GIT_MIRROR/fdroid/repo/com.politedroid_4.apk
|
||||
test -e $GIT_MIRROR/fdroid/repo/com.politedroid_5.apk
|
||||
test -e $GIT_MIRROR/fdroid/repo/com.politedroid_6.apk
|
||||
test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_4.apk
|
||||
test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_5.apk
|
||||
test -e $SERVER_GIT_MIRROR/fdroid/repo/com.politedroid_6.apk
|
||||
before=`du -s --bytes $GIT_MIRROR/.git/ | awk '{print $1}'`
|
||||
|
||||
echo "git_mirror_size_limit = '60kb'" >> config.py
|
||||
$fdroid update
|
||||
$fdroid deploy
|
||||
test -e $REPOROOT/archive/com.politedroid_3.apk
|
||||
! test -e $SERVER_GIT_MIRROR/fdroid/archive/com.politedroid_3.apk
|
||||
after=`du -s --bytes $GIT_MIRROR/.git/ | awk '{print $1}'`
|
||||
! test -e $GIT_MIRROR/.git/test-stamp
|
||||
git -C $GIT_MIRROR gc
|
||||
git -C $SERVER_GIT_MIRROR gc
|
||||
test $before -gt $after
|
||||
|
||||
|
||||
#------------------------------------------------------------------------------#
|
||||
echo_header "sign binary repo in offline box, then publishing from online box"
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue