refactor update.py:main
This makes update.py:main a lot saner by removing a lot of the implicit assumptions between the different stages of generating the repository index. * mostly unify repo and archive processing, that means the archive is now actually getting the same treatment regarding i.e. fastlane data. Previously the archive didn't get considered at all here. * already filter the list of apps to include in a repo in update.py and give that prefiltered list to index. This makes sure we actually only copy fastlane/triple-t/etc. stuff for apps ending up in the index. This both, can save a lot of time if there are a lot of old /build dirs lying around and doesn't clutter /repo with things that aren't referenced from the index. Closes fdroid/fdroidserver#524
This commit is contained in:
Marcus Hoffmann
parent
e66683720b
commit
d720c99ae5
|
|
@ -21,7 +21,6 @@
|
|||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import collections
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
|
@ -41,7 +40,7 @@ from . import metadata
|
|||
from . import net
|
||||
from . import signindex
|
||||
from fdroidserver.common import FDroidPopen, FDroidPopenBytes, load_stats_fdroid_signing_key_fingerprints
|
||||
from fdroidserver.exception import FDroidException, VerificationException, MetaDataException
|
||||
from fdroidserver.exception import FDroidException, VerificationException
|
||||
|
||||
|
||||
def make(apps, apks, repodir, archive):
|
||||
|
|
@ -49,8 +48,9 @@ def make(apps, apks, repodir, archive):
|
|||
|
||||
This requires properly initialized options and config objects.
|
||||
|
||||
:param apps: fully populated apps list
|
||||
:param apks: full populated apks list
|
||||
:param apps: OrderedDict of apps to go into the index, each app should have
|
||||
at least one associated apk
|
||||
:param apks: list of apks to go into the index
|
||||
:param repodir: the repo directory
|
||||
:param archive: True if this is the archive repo, False if it's the
|
||||
main one.
|
||||
|
|
@ -60,6 +60,12 @@ def make(apps, apks, repodir, archive):
|
|||
if not common.options.nosign:
|
||||
common.assert_config_keystore(common.config)
|
||||
|
||||
# Historically the index has been sorted by App Name, so we enforce this ordering here
|
||||
sortedids = sorted(apps, key=lambda appid: apps[appid].Name.upper())
|
||||
sortedapps = collections.OrderedDict()
|
||||
for appid in sortedids:
|
||||
sortedapps[appid] = apps[appid]
|
||||
|
||||
repodict = collections.OrderedDict()
|
||||
repodict['timestamp'] = datetime.utcnow().replace(tzinfo=timezone.utc)
|
||||
repodict['version'] = METADATA_VERSION
|
||||
|
|
@ -100,24 +106,6 @@ def make(apps, apks, repodir, archive):
|
|||
if mirrors:
|
||||
repodict['mirrors'] = mirrors
|
||||
|
||||
# Historically the index has been sorted by App Name, so we enforce this ordering here
|
||||
sortedids = sorted(apps, key=lambda appid: apps[appid].Name.upper())
|
||||
|
||||
appsWithPackages = collections.OrderedDict()
|
||||
for packageName in sortedids:
|
||||
app = apps[packageName]
|
||||
if app['Disabled']:
|
||||
continue
|
||||
|
||||
# only include apps with packages
|
||||
for apk in apks:
|
||||
if apk['packageName'] == packageName:
|
||||
newapp = copy.copy(app) # update wiki needs unmodified description
|
||||
newapp['Description'] = metadata.description_html(app['Description'],
|
||||
metadata.DescriptionResolver(apps))
|
||||
appsWithPackages[packageName] = newapp
|
||||
break
|
||||
|
||||
requestsdict = collections.OrderedDict()
|
||||
for command in ('install', 'uninstall'):
|
||||
packageNames = []
|
||||
|
|
@ -133,9 +121,9 @@ def make(apps, apks, repodir, archive):
|
|||
|
||||
fdroid_signing_key_fingerprints = load_stats_fdroid_signing_key_fingerprints()
|
||||
|
||||
make_v0(appsWithPackages, apks, repodir, repodict, requestsdict,
|
||||
make_v0(sortedapps, apks, repodir, repodict, requestsdict,
|
||||
fdroid_signing_key_fingerprints)
|
||||
make_v1(appsWithPackages, apks, repodir, repodict, requestsdict,
|
||||
make_v1(sortedapps, apks, repodir, repodict, requestsdict,
|
||||
fdroid_signing_key_fingerprints)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -2166,6 +2166,71 @@ def create_metadata_from_template(apk):
|
|||
logging.info(_("Generated skeleton metadata for {appid}").format(appid=apk['packageName']))
|
||||
|
||||
|
||||
def read_names_from_apks(apps, apks):
|
||||
"""This is a stripped down copy of apply_info_from_latest_apk that only parses app names"""
|
||||
for appid, app in apps.items():
|
||||
bestver = UNSET_VERSION_CODE
|
||||
for apk in apks:
|
||||
if apk['packageName'] == appid:
|
||||
if apk['versionCode'] > bestver:
|
||||
bestver = apk['versionCode']
|
||||
bestapk = apk
|
||||
|
||||
if bestver == UNSET_VERSION_CODE:
|
||||
if app.Name is None:
|
||||
app.Name = app.AutoName or appid
|
||||
app.icon = None
|
||||
else:
|
||||
if app.Name is None:
|
||||
app.Name = bestapk['name']
|
||||
|
||||
|
||||
def render_app_descriptions(apps, all_apps):
|
||||
"""
|
||||
Renders the app html description.
|
||||
For resolving inter-app links it needs the full list of apps, even if they end up in
|
||||
separate repos (i.e. archive or per app repos).
|
||||
"""
|
||||
for app in apps.values():
|
||||
app['Description'] = metadata.description_html(app['Description'], metadata.DescriptionResolver(all_apps))
|
||||
|
||||
|
||||
def get_apps_with_packages(apps, apks):
|
||||
"""Returns a deepcopy of that subset apps that actually has any associated packages. Skips disabled apps."""
|
||||
appsWithPackages = collections.OrderedDict()
|
||||
for packageName in apps:
|
||||
app = apps[packageName]
|
||||
if app['Disabled']:
|
||||
continue
|
||||
|
||||
# only include apps with packages
|
||||
for apk in apks:
|
||||
if apk['packageName'] == packageName:
|
||||
newapp = copy.copy(app)
|
||||
appsWithPackages[packageName] = newapp
|
||||
break
|
||||
return appsWithPackages
|
||||
|
||||
|
||||
def prepare_apps(apps, apks, repodir):
|
||||
"""Encapsulates all necessary preparation steps before we can build an index out of apps and apks.
|
||||
|
||||
:param apps: All apps as read from metadata
|
||||
:param apks: list of apks that belong into repo, this gets modified in place
|
||||
:param repodir: the target repository directory, metadata files will be copied here
|
||||
:return: the relevant subset of apps (as a deepcopy)
|
||||
"""
|
||||
apps_with_packages = get_apps_with_packages(apps, apks)
|
||||
apply_info_from_latest_apk(apps_with_packages, apks)
|
||||
render_app_descriptions(apps_with_packages, apps)
|
||||
insert_funding_yml_donation_links(apps)
|
||||
copy_triple_t_store_metadata(apps_with_packages, repodir)
|
||||
insert_obbs(repodir, apps_with_packages, apks)
|
||||
translate_per_build_anti_features(apps_with_packages, apks)
|
||||
insert_localized_app_metadata(apps_with_packages, repodir)
|
||||
return apps_with_packages
|
||||
|
||||
|
||||
config = None
|
||||
options = None
|
||||
start_timestamp = time.gmtime()
|
||||
|
|
@ -2302,12 +2367,6 @@ def main():
|
|||
else:
|
||||
logging.warning(msg + '\n\t' + _('Use `fdroid update -c` to create it.'))
|
||||
|
||||
insert_funding_yml_donation_links(apps)
|
||||
copy_triple_t_store_metadata(apps, repodirs[0])
|
||||
insert_obbs(repodirs[0], apps, apks)
|
||||
insert_localized_app_metadata(apps, repodirs[0])
|
||||
translate_per_build_anti_features(apps, apks)
|
||||
|
||||
# Scan the archive repo for apks as well
|
||||
if len(repodirs) > 1:
|
||||
archapks, cc = process_apks(apkcache, repodirs[1], knownapks, options.use_date_from_apk)
|
||||
|
|
@ -2316,8 +2375,18 @@ def main():
|
|||
else:
|
||||
archapks = []
|
||||
|
||||
# Apply information from latest apks to the application and update dates
|
||||
apply_info_from_latest_apk(apps, apks + archapks)
|
||||
# We need app.Name populated for all apps regardless of which repo they end up in
|
||||
# for the old-style inter-app links, so let's do it before we do anything else.
|
||||
# This will be done again (as part of apply_info_from_latest_apk) for repo and archive
|
||||
# separately later on, but it's fairly cheap anyway.
|
||||
read_names_from_apks(apps, apks + archapks)
|
||||
|
||||
if len(repodirs) > 1:
|
||||
archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
|
||||
archived_apps = prepare_apps(apps, archapks, repodirs[1])
|
||||
index.make(archived_apps, archapks, repodirs[1], True)
|
||||
|
||||
repoapps = prepare_apps(apps, apks, repodirs[0])
|
||||
|
||||
# APKs are placed into multiple repos based on the app package, providing
|
||||
# per-app subscription feeds for nightly builds and things like it
|
||||
|
|
@ -2333,20 +2402,10 @@ def main():
|
|||
logging.info(_('Skipping index generation for {appid}').format(appid=appid))
|
||||
return
|
||||
|
||||
if len(repodirs) > 1:
|
||||
archive_old_apks(apps, apks, archapks, repodirs[0], repodirs[1], config['archive_older'])
|
||||
|
||||
# Make the index for the main repo...
|
||||
index.make(apps, apks, repodirs[0], False)
|
||||
index.make(repoapps, apks, repodirs[0], False)
|
||||
make_categories_txt(repodirs[0], categories)
|
||||
|
||||
# If there's an archive repo, make the index for it. We already scanned it
|
||||
# earlier on.
|
||||
if len(repodirs) > 1:
|
||||
archived_apps = copy.deepcopy(apps)
|
||||
apply_info_from_latest_apk(archived_apps, archapks)
|
||||
index.make(archived_apps, archapks, repodirs[1], True)
|
||||
|
||||
git_remote = config.get('binary_transparency_remote')
|
||||
if git_remote or os.path.isdir(os.path.join('binary_transparency', '.git')):
|
||||
from . import btlog
|
||||
|
|
|
|||
Loading…
Reference in New Issue