diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1ed94e3..6581b1e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: debian:buster +image: debian:bullseye stages: - test @@ -34,10 +34,8 @@ variables: python3-bleach python3-cryptography python3-django-allauth - python3-django-compat python3-django-compressor - python3-django-hvad - python3-django-js-reverse + python3-django-modeltranslation python3-django-sass-processor python3-dockerpycreds python3-libcloud @@ -55,11 +53,11 @@ variables: rsync -pep8: +pycodestyle: stage: test script: - *apt-template - - apt-get install pep8 + - apt-get install pycodestyle - ./tests/test-pep8.sh pylint: @@ -99,11 +97,11 @@ docker: - echo $CI_BUILD_TOKEN | docker login -u gitlab-ci-token --password-stdin registry.gitlab.com script: - docker build -t $CI_REGISTRY_IMAGE:latest . - - docker tag $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:buster - - docker tag $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:django-1.11 + - docker tag $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:bullseye + - docker tag $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:django-2 - docker push $CI_REGISTRY_IMAGE:latest - - docker push $CI_REGISTRY_IMAGE:buster - - docker push $CI_REGISTRY_IMAGE:django-1.11 + - docker push $CI_REGISTRY_IMAGE:bullseye + - docker push $CI_REGISTRY_IMAGE:django-2 when: on_success only: diff --git a/.pylintrc b/.pylintrc index 9a47f80..54a21fe 100644 --- a/.pylintrc +++ b/.pylintrc @@ -112,12 +112,6 @@ max-line-length=100 # Maximum number of lines in a module max-module-lines=1000 -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma,dict-separator - # Allow the body of a class to be on the same line as the declaration if body # contains single statement. single-line-class-stmt=no @@ -180,10 +174,6 @@ ignored-classes=optparse.Values,thread._local,_thread._local # supports qualified module names, as well as Unix pattern matching. ignored-modules= -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - # The minimum edit distance a name should have in order to be considered a # similar match for a missing member name. missing-member-hint-distance=1 @@ -252,36 +242,21 @@ redefining-builtins-modules=six.moves,future.builtins [BASIC] -# Naming hint for argument names -argument-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - # Regular expression matching correct argument names argument-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ -# Naming hint for attribute names -attr-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - # Regular expression matching correct attribute names attr-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ # Bad variable names which should always be refused, separated by a comma bad-names=foo,bar,baz,toto,tutu,tata -# Naming hint for class attribute names -class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - # Regular expression matching correct class attribute names class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ -# Naming hint for class names -class-name-hint=[A-Z_][a-zA-Z0-9]+$ - # Regular expression matching correct class names class-rgx=[A-Z_][a-zA-Z0-9]+$ -# Naming hint for constant names -const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - # Regular expression matching correct constant names const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__)|(urls|urlpatterns|register))$ @@ -289,9 +264,6 @@ const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__)|(urls|urlpatterns|register))$ # ones are exempt. docstring-min-length=-1 -# Naming hint for function names -function-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - # Regular expression matching correct function names function-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ @@ -301,21 +273,12 @@ good-names=i,j,k,ex,Run,_,qs # Include a hint for the correct naming format with invalid-name include-naming-hint=no -# Naming hint for inline iteration names -inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ - # Regular expression matching correct inline iteration names inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ -# Naming hint for method names -method-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - # Regular expression matching correct method names method-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ -# Naming hint for module names -module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - # Regular expression matching correct module names module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ @@ -331,9 +294,6 @@ no-docstring-rgx=^_ # to this list to register other decorators that produce valid properties. property-classes=abc.abstractproperty -# Naming hint for variable names -variable-name-hint=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - # Regular expression matching correct variable names variable-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ diff --git a/Dockerfile b/Dockerfile index fa22cfb..e1d9b80 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:buster +FROM debian:bullseye MAINTAINER team@f-droid.org ENV PYTHONUNBUFFERED 1 @@ -24,10 +24,10 @@ RUN echo Etc/UTC > /etc/timezone \ 'APT::Get::Assume-Yes "true";' \ 'Dpkg::Use-Pty "0";'\ > /etc/apt/apt.conf.d/99headless \ - && printf "Package: apksigner libapksig-java fdroidserver s3cmd\nPin: release a=buster-backports\nPin-Priority: 500\n" \ - > /etc/apt/preferences.d/buster-backports.pref \ - && echo "deb http://deb.debian.org/debian/ buster-backports main" \ - > /etc/apt/sources.list.d/buster-backports.list + && printf "Package: apksigner libapksig-java\nPin: release a=bullseye-backports\nPin-Priority: 500\n" \ + > /etc/apt/preferences.d/bullseye-backports.pref \ + && echo "deb https://deb.debian.org/debian/ bullseye-backports main" \ + > /etc/apt/sources.list.d/bullseye-backports.list # a version of the Debian package list is also in .gitlab-ci.yml RUN apt-get update && apt-get dist-upgrade && apt-get install \ @@ -46,10 +46,8 @@ RUN apt-get update && apt-get dist-upgrade && apt-get install \ python3-cryptography \ python3-dev \ python3-django-allauth \ - python3-django-compat \ python3-django-compressor \ - python3-django-hvad \ - python3-django-js-reverse \ + python3-django-modeltranslation \ python3-django-sass-processor \ python3-dockerpycreds \ python3-libcloud \ diff --git a/Vagrantfile b/Vagrantfile index 5d5843e..b8e43d9 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -30,12 +30,16 @@ end before_script_file.rewind Vagrant.configure("2") do |config| - config.vm.box = "fdroid/basebox-buster64" + config.vm.box = "debian/bullseye64" config.vm.network "forwarded_port", guest: 8000, host: 8000 config.vm.synced_folder '.', '/vagrant', disabled: true config.vm.provision "file", source: env_file.path, destination: 'env.sh' config.vm.provision :shell, inline: <<-SHELL set -ex + + apt-get update + apt-get -qy install git + mv ~vagrant/env.sh #{sourcepath} source #{sourcepath} mkdir -p $(dirname $CI_PROJECT_DIR) diff --git a/repomaker/admin.py b/repomaker/admin.py index 788ff80..9a68437 100644 --- a/repomaker/admin.py +++ b/repomaker/admin.py @@ -1,5 +1,5 @@ from django.contrib import admin -from hvad.admin import TranslatableAdmin +from modeltranslation.admin import TranslationAdmin from .models import Repository, RemoteRepository, App, RemoteApp, Apk, ApkPointer, \ RemoteApkPointer, Category, Screenshot, RemoteScreenshot @@ -7,8 +7,8 @@ from .models.storage import StorageManager admin.site.register(Repository) admin.site.register(RemoteRepository) -admin.site.register(App, TranslatableAdmin) # hides untranslated apps which should not exist -admin.site.register(RemoteApp, TranslatableAdmin) # hides untranslated apps which should not exist +admin.site.register(App, TranslationAdmin) # hides untranslated apps which should not exist +admin.site.register(RemoteApp, TranslationAdmin) # hides untranslated apps which should not exist admin.site.register(Apk) admin.site.register(ApkPointer) admin.site.register(RemoteApkPointer) diff --git a/repomaker/gui.py b/repomaker/gui.py index f181ade..8127c72 100644 --- a/repomaker/gui.py +++ b/repomaker/gui.py @@ -28,7 +28,7 @@ def create_window(): global terminate # pylint: disable=global-statement try: webview.config["USE_QT"] = True # use Qt instead of Gtk for webview - webview.create_window("Repomaker", confirm_quit=True) + webview.create_window("Repomaker", confirm_close=True) terminate = True finally: # halt background tasks @@ -82,6 +82,6 @@ def get_loading_screen(): def server_started(): try: - return requests.head(URL).status_code == requests.codes.OK + return requests.head(URL, timeout=60).status_code == requests.codes.OK except Exception: return False diff --git a/repomaker/migrations/0001_initial.py b/repomaker/migrations/0001_initial.py index 81a747d..17c25a9 100644 --- a/repomaker/migrations/0001_initial.py +++ b/repomaker/migrations/0001_initial.py @@ -1,14 +1,11 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.5 on 2017-09-29 19:46 -from __future__ import unicode_literals +# Generated by Django 2.0.13 on 2022-05-23 22:41 +from django.conf import settings +from django.db import migrations, models import django.db.models.deletion -import django.db.models.manager import django.utils.timezone import repomaker.models.storage import repomaker.storage -from django.conf import settings -from django.db import migrations, models class Migration(migrations.Migration): @@ -59,36 +56,48 @@ class Migration(migrations.Migration): ('website', models.URLField(blank=True, max_length=2048)), ('icon', models.ImageField(upload_to=repomaker.storage.get_icon_file_path_for_app)), ('added_date', models.DateTimeField(default=django.utils.timezone.now)), + ('summary', models.CharField(blank=True, max_length=255)), + ('summary_en_us', models.CharField(blank=True, max_length=255, null=True)), + ('summary_en', models.CharField(blank=True, max_length=255, null=True)), + ('summary_de_de', models.CharField(blank=True, max_length=255, null=True)), + ('summary_de', models.CharField(blank=True, max_length=255, null=True)), + ('summary_fr', models.CharField(blank=True, max_length=255, null=True)), + ('summary_zh_cn', models.CharField(blank=True, max_length=255, null=True)), + ('description', models.TextField(blank=True)), + ('description_en_us', models.TextField(blank=True, null=True)), + ('description_en', models.TextField(blank=True, null=True)), + ('description_de_de', models.TextField(blank=True, null=True)), + ('description_de', models.TextField(blank=True, null=True)), + ('description_fr', models.TextField(blank=True, null=True)), + ('description_zh_cn', models.TextField(blank=True, null=True)), + ('available_languages', models.TextField(default='', max_length=8)), ('type', models.CharField(choices=[('apk', 'APK'), ('book', 'Book'), ('document', 'Document'), ('image', 'Image'), ('audio', 'Audio'), ('video', 'Video'), ('other', 'Other')], default='apk', max_length=16)), ('last_updated_date', models.DateTimeField(auto_now=True)), - ], - options={ - 'abstract': False, - 'ordering': ['added_date'], - }, - managers=[ - ('objects', django.db.models.manager.Manager()), - ('_plain_manager', django.db.models.manager.Manager()), - ], - ), - migrations.CreateModel( - name='AppTranslation', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('summary', models.CharField(blank=True, max_length=255)), - ('description', models.TextField(blank=True)), ('feature_graphic', models.ImageField(blank=True, max_length=1024, upload_to=repomaker.storage.get_graphic_asset_file_path)), + ('feature_graphic_en_us', models.ImageField(blank=True, max_length=1024, null=True, upload_to=repomaker.storage.get_graphic_asset_file_path)), + ('feature_graphic_en', models.ImageField(blank=True, max_length=1024, null=True, upload_to=repomaker.storage.get_graphic_asset_file_path)), + ('feature_graphic_de_de', models.ImageField(blank=True, max_length=1024, null=True, upload_to=repomaker.storage.get_graphic_asset_file_path)), + ('feature_graphic_de', models.ImageField(blank=True, max_length=1024, null=True, upload_to=repomaker.storage.get_graphic_asset_file_path)), + ('feature_graphic_fr', models.ImageField(blank=True, max_length=1024, null=True, upload_to=repomaker.storage.get_graphic_asset_file_path)), + ('feature_graphic_zh_cn', models.ImageField(blank=True, max_length=1024, null=True, upload_to=repomaker.storage.get_graphic_asset_file_path)), ('high_res_icon', models.ImageField(blank=True, max_length=1024, upload_to=repomaker.storage.get_graphic_asset_file_path)), + ('high_res_icon_en_us', models.ImageField(blank=True, max_length=1024, null=True, upload_to=repomaker.storage.get_graphic_asset_file_path)), + ('high_res_icon_en', models.ImageField(blank=True, max_length=1024, null=True, upload_to=repomaker.storage.get_graphic_asset_file_path)), + ('high_res_icon_de_de', models.ImageField(blank=True, max_length=1024, null=True, upload_to=repomaker.storage.get_graphic_asset_file_path)), + ('high_res_icon_de', models.ImageField(blank=True, max_length=1024, null=True, upload_to=repomaker.storage.get_graphic_asset_file_path)), + ('high_res_icon_fr', models.ImageField(blank=True, max_length=1024, null=True, upload_to=repomaker.storage.get_graphic_asset_file_path)), + ('high_res_icon_zh_cn', models.ImageField(blank=True, max_length=1024, null=True, upload_to=repomaker.storage.get_graphic_asset_file_path)), ('tv_banner', models.ImageField(blank=True, max_length=1024, upload_to=repomaker.storage.get_graphic_asset_file_path)), - ('language_code', models.CharField(db_index=True, max_length=15)), - ('master', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='translations', to='repomaker.App')), + ('tv_banner_en_us', models.ImageField(blank=True, max_length=1024, null=True, upload_to=repomaker.storage.get_graphic_asset_file_path)), + ('tv_banner_en', models.ImageField(blank=True, max_length=1024, null=True, upload_to=repomaker.storage.get_graphic_asset_file_path)), + ('tv_banner_de_de', models.ImageField(blank=True, max_length=1024, null=True, upload_to=repomaker.storage.get_graphic_asset_file_path)), + ('tv_banner_de', models.ImageField(blank=True, max_length=1024, null=True, upload_to=repomaker.storage.get_graphic_asset_file_path)), + ('tv_banner_fr', models.ImageField(blank=True, max_length=1024, null=True, upload_to=repomaker.storage.get_graphic_asset_file_path)), + ('tv_banner_zh_cn', models.ImageField(blank=True, max_length=1024, null=True, upload_to=repomaker.storage.get_graphic_asset_file_path)), ], options={ + 'ordering': ['added_date'], 'abstract': False, - 'db_table': 'repomaker_app_translation', - 'managed': True, - 'db_tablespace': '', - 'default_permissions': (), }, ), migrations.CreateModel( @@ -140,40 +149,70 @@ class Migration(migrations.Migration): ('website', models.URLField(blank=True, max_length=2048)), ('icon', models.ImageField(upload_to=repomaker.storage.get_icon_file_path_for_app)), ('added_date', models.DateTimeField(default=django.utils.timezone.now)), + ('summary', models.CharField(blank=True, max_length=255)), + ('summary_en_us', models.CharField(blank=True, max_length=255, null=True)), + ('summary_en', models.CharField(blank=True, max_length=255, null=True)), + ('summary_de_de', models.CharField(blank=True, max_length=255, null=True)), + ('summary_de', models.CharField(blank=True, max_length=255, null=True)), + ('summary_fr', models.CharField(blank=True, max_length=255, null=True)), + ('summary_zh_cn', models.CharField(blank=True, max_length=255, null=True)), + ('description', models.TextField(blank=True)), + ('description_en_us', models.TextField(blank=True, null=True)), + ('description_en', models.TextField(blank=True, null=True)), + ('description_de_de', models.TextField(blank=True, null=True)), + ('description_de', models.TextField(blank=True, null=True)), + ('description_fr', models.TextField(blank=True, null=True)), + ('description_zh_cn', models.TextField(blank=True, null=True)), + ('available_languages', models.TextField(default='', max_length=8)), ('icon_etag', models.CharField(blank=True, max_length=128, null=True)), ('last_updated_date', models.DateTimeField(blank=True)), - ('category', models.ManyToManyField(blank=True, to='repomaker.Category')), - ], - options={ - 'abstract': False, - 'ordering': ['added_date'], - }, - managers=[ - ('objects', django.db.models.manager.Manager()), - ('_plain_manager', django.db.models.manager.Manager()), - ], - ), - migrations.CreateModel( - name='RemoteAppTranslation', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('summary', models.CharField(blank=True, max_length=255)), - ('description', models.TextField(blank=True)), ('feature_graphic_url', models.URLField(blank=True, max_length=2048)), + ('feature_graphic_url_en_us', models.URLField(blank=True, max_length=2048, null=True)), + ('feature_graphic_url_en', models.URLField(blank=True, max_length=2048, null=True)), + ('feature_graphic_url_de_de', models.URLField(blank=True, max_length=2048, null=True)), + ('feature_graphic_url_de', models.URLField(blank=True, max_length=2048, null=True)), + ('feature_graphic_url_fr', models.URLField(blank=True, max_length=2048, null=True)), + ('feature_graphic_url_zh_cn', models.URLField(blank=True, max_length=2048, null=True)), ('feature_graphic_etag', models.CharField(blank=True, max_length=128, null=True)), + ('feature_graphic_etag_en_us', models.CharField(blank=True, max_length=128, null=True)), + ('feature_graphic_etag_en', models.CharField(blank=True, max_length=128, null=True)), + ('feature_graphic_etag_de_de', models.CharField(blank=True, max_length=128, null=True)), + ('feature_graphic_etag_de', models.CharField(blank=True, max_length=128, null=True)), + ('feature_graphic_etag_fr', models.CharField(blank=True, max_length=128, null=True)), + ('feature_graphic_etag_zh_cn', models.CharField(blank=True, max_length=128, null=True)), ('high_res_icon_url', models.URLField(blank=True, max_length=2048)), + ('high_res_icon_url_en_us', models.URLField(blank=True, max_length=2048, null=True)), + ('high_res_icon_url_en', models.URLField(blank=True, max_length=2048, null=True)), + ('high_res_icon_url_de_de', models.URLField(blank=True, max_length=2048, null=True)), + ('high_res_icon_url_de', models.URLField(blank=True, max_length=2048, null=True)), + ('high_res_icon_url_fr', models.URLField(blank=True, max_length=2048, null=True)), + ('high_res_icon_url_zh_cn', models.URLField(blank=True, max_length=2048, null=True)), ('high_res_icon_etag', models.CharField(blank=True, max_length=128, null=True)), + ('high_res_icon_etag_en_us', models.CharField(blank=True, max_length=128, null=True)), + ('high_res_icon_etag_en', models.CharField(blank=True, max_length=128, null=True)), + ('high_res_icon_etag_de_de', models.CharField(blank=True, max_length=128, null=True)), + ('high_res_icon_etag_de', models.CharField(blank=True, max_length=128, null=True)), + ('high_res_icon_etag_fr', models.CharField(blank=True, max_length=128, null=True)), + ('high_res_icon_etag_zh_cn', models.CharField(blank=True, max_length=128, null=True)), ('tv_banner_url', models.URLField(blank=True, max_length=2048)), + ('tv_banner_url_en_us', models.URLField(blank=True, max_length=2048, null=True)), + ('tv_banner_url_en', models.URLField(blank=True, max_length=2048, null=True)), + ('tv_banner_url_de_de', models.URLField(blank=True, max_length=2048, null=True)), + ('tv_banner_url_de', models.URLField(blank=True, max_length=2048, null=True)), + ('tv_banner_url_fr', models.URLField(blank=True, max_length=2048, null=True)), + ('tv_banner_url_zh_cn', models.URLField(blank=True, max_length=2048, null=True)), ('tv_banner_etag', models.CharField(blank=True, max_length=128, null=True)), - ('language_code', models.CharField(db_index=True, max_length=15)), - ('master', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='translations', to='repomaker.RemoteApp')), + ('tv_banner_etag_en_us', models.CharField(blank=True, max_length=128, null=True)), + ('tv_banner_etag_en', models.CharField(blank=True, max_length=128, null=True)), + ('tv_banner_etag_de_de', models.CharField(blank=True, max_length=128, null=True)), + ('tv_banner_etag_de', models.CharField(blank=True, max_length=128, null=True)), + ('tv_banner_etag_fr', models.CharField(blank=True, max_length=128, null=True)), + ('tv_banner_etag_zh_cn', models.CharField(blank=True, max_length=128, null=True)), + ('category', models.ManyToManyField(blank=True, limit_choices_to={'user': None}, to='repomaker.Category')), ], options={ + 'ordering': ['added_date'], 'abstract': False, - 'db_table': 'repomaker_remoteapp_translation', - 'managed': True, - 'db_tablespace': '', - 'default_permissions': (), }, ), migrations.CreateModel( @@ -303,7 +342,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='app', name='category', - field=models.ManyToManyField(blank=True, to='repomaker.Category'), + field=models.ManyToManyField(blank=True, limit_choices_to={'user': None}, to='repomaker.Category'), ), migrations.AddField( model_name='app', @@ -325,32 +364,24 @@ class Migration(migrations.Migration): name='repo', field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='repomaker.Repository'), ), - migrations.AlterUniqueTogether( - name='remoteapptranslation', - unique_together=set([('language_code', 'master')]), - ), migrations.AlterUniqueTogether( name='remoteapp', - unique_together=set([('package_id', 'repo')]), + unique_together={('package_id', 'repo')}, ), migrations.AlterUniqueTogether( name='remoteapkpointer', - unique_together=set([('apk', 'app')]), + unique_together={('apk', 'app')}, ), migrations.AlterUniqueTogether( name='category', - unique_together=set([('user', 'name')]), - ), - migrations.AlterUniqueTogether( - name='apptranslation', - unique_together=set([('language_code', 'master')]), + unique_together={('user', 'name')}, ), migrations.AlterUniqueTogether( name='app', - unique_together=set([('package_id', 'repo')]), + unique_together={('package_id', 'repo')}, ), migrations.AlterUniqueTogether( name='apkpointer', - unique_together=set([('apk', 'app')]), + unique_together={('apk', 'app')}, ), ] diff --git a/repomaker/migrations/default_remote_repositories.py b/repomaker/migrations/default_remote_repositories.py index c06e6eb..8e30414 100644 --- a/repomaker/migrations/default_remote_repositories.py +++ b/repomaker/migrations/default_remote_repositories.py @@ -24,7 +24,7 @@ def forwards_func(apps, schema_editor): last_change_date=datetime.datetime.fromtimestamp(0, timezone.utc), update_scheduled=True, ) - repo.users = User.objects.all() + repo.users.set(User.objects.all()) repo.save() tasks.update_remote_repo(repo.pk, repeat=Task.DAILY, priority=tasks.PRIORITY_REMOTE_REPO) @@ -38,7 +38,7 @@ def forwards_func(apps, schema_editor): last_change_date=datetime.datetime.fromtimestamp(0, timezone.utc), update_scheduled=True, ) - repo.users = User.objects.all() + repo.users.set(User.objects.all()) repo.save() tasks.update_remote_repo(repo.pk, repeat=Task.DAILY, priority=tasks.PRIORITY_REMOTE_REPO) diff --git a/repomaker/models/apk.py b/repomaker/models/apk.py index 0fa31d5..67931c5 100644 --- a/repomaker/models/apk.py +++ b/repomaker/models/apk.py @@ -1,3 +1,5 @@ +import fdroidserver +import hashlib import logging import os import zipfile @@ -12,7 +14,6 @@ from django.db.models.signals import post_delete, pre_delete from django.dispatch import receiver from django.utils import timezone from django.utils.translation import ugettext_lazy as _ -from fdroidserver import common, exception, update from repomaker import tasks from repomaker.models.repository import AbstractRepository @@ -55,7 +56,7 @@ class Apk(models.Model): # download and store file file_name = url.rsplit('/', 1)[-1] - r = requests.get(url) + r = requests.get(url, timeout=60) if r.status_code != requests.codes.ok: # TODO delete self and ApkPointer when this fails permanently r.raise_for_status() @@ -95,7 +96,7 @@ class Apk(models.Model): if ext == '.apk': try: repo_file = self._get_info_from_apk() - except exception.BuildException as e: + except fdroidserver.exception.BuildException as e: raise ValidationError(e) except zipfile.BadZipFile as e: raise ValidationError(e) @@ -143,14 +144,14 @@ class Apk(models.Model): AbstractRepository().get_config() # Verify that the signature is correct - if not common.verify_apk_signature(self.file.path): + if not fdroidserver.verify_apk_signature(self.file.path): raise ValidationError(_('Invalid APK signature')) # scan APK and extract information about it try: - repo_file = update.scan_apk(self.file.path) + repo_file = fdroidserver.scan_apk(self.file.path) repo_file['type'] = APK - except exception.BuildException as e: + except fdroidserver.exception.BuildException as e: raise ValidationError(e) if 'packageName' not in repo_file: @@ -161,13 +162,13 @@ class Apk(models.Model): def _get_info_from_file(self): repo_file = { 'sig': None, - 'hash': update.sha256sum(self.file.path), + 'hash': sha256sum(self.file.path), 'hashType': 'sha256', 'size': self.file.size, 'type': self._get_type() } file_name = os.path.basename(self.file.name) - match = common.STANDARD_FILE_NAME_REGEX.match(file_name) + match = fdroidserver.common.STANDARD_FILE_NAME_REGEX.match(file_name) if match: repo_file['packageName'] = match.group(1) repo_file['versionName'] = match.group(2) @@ -248,3 +249,15 @@ def apk_post_delete_handler(**kwargs): if apk.file: logging.info("Deleting APK: %s", apk.file.name) apk.file.delete(save=False) + + +def sha256sum(filename): + """Calculate the sha256 of the given file.""" + sha = hashlib.sha256() + with open(filename, 'rb') as f: + while True: + t = f.read(16384) + if len(t) == 0: + break + sha.update(t) + return sha.hexdigest() diff --git a/repomaker/models/app.py b/repomaker/models/app.py index 8ccf2cd..4edb882 100644 --- a/repomaker/models/app.py +++ b/repomaker/models/app.py @@ -7,11 +7,10 @@ from django.db.models.signals import post_delete from django.dispatch import receiver from django.templatetags.static import static from django.urls import reverse -from django.utils import timezone -from django.utils.translation import ugettext_lazy as _, get_language +from django.utils import timezone, translation +from django.utils.translation import ugettext_lazy as _ from fdroidserver import metadata, net -from hvad.models import TranslatableModel, TranslatedFields -from hvad.utils import load_translation +from modeltranslation.utils import get_language from repomaker.storage import get_icon_file_path_for_app, \ get_graphic_asset_file_path @@ -38,7 +37,7 @@ TYPE_CHOICES = ( APP_DEFAULT_ICON = os.path.join('repomaker', 'images', 'default-app-icon.png') -class AbstractApp(TranslatableModel): +class AbstractApp(models.Model): package_id = models.CharField(max_length=255, blank=True) name = models.CharField(max_length=255, blank=True) summary_override = models.CharField(max_length=255, blank=True) @@ -48,11 +47,15 @@ class AbstractApp(TranslatableModel): icon = models.ImageField(upload_to=get_icon_file_path_for_app) category = models.ManyToManyField(Category, blank=True, limit_choices_to={'user': None}) added_date = models.DateTimeField(default=timezone.now) - translations = TranslatedFields( - # for historic reasons summary and description are also included non-localized in the index - summary=models.CharField(max_length=255, blank=True), - description=models.TextField(blank=True), # always clean and then consider safe - ) + + # Translated fields + # for historic reasons summary and description are also included non-localized in the index + summary = models.CharField(max_length=255, blank=True) + description = models.TextField(blank=True) # always clean and then consider safe + # hvad kept track of what translations exist, modeltranslation doesn't so we + # keep track in this field (max_lenght just affects admin display) + # we keep a comma separated list + available_languages = models.TextField(max_length=8, default="") def __str__(self): return self.name @@ -63,23 +66,37 @@ class AbstractApp(TranslatableModel): return self.icon.url return static(APP_DEFAULT_ICON) + def translate(self, language): + """ + This was a method from hvad, we're keeping track becasue modeltranslation + doesn't + + order of the list is significant, the first item is assumed to be the + default language + """ + + if self.available_languages == "": + self.available_languages = language + else: + lang_list = self.available_languages.split(',') + lang_list.append(language) + lang_list = list(set(lang_list)) # remove duplicate values + self.available_languages = ','.join(lang_list) + def default_translate(self): """Creates a new default translation""" - language = get_language() - if language is None: - self.translate(settings.LANGUAGE_CODE) - else: - self.translate(language) + self.translate(get_language()) - def get_translation(self, language_code=get_language()): + def get_available_languages(self): """ - Returns a translation of this instance for the given language_code + This method was originally provided by django-hvad - A valid translation instance is always returned. - It will be loaded from the database as required. - If this fails, a new, empty, ready-to-use translation will be returned. + TODO: should we just keep track of what translations are provided + in a separate field? """ - return load_translation(self, language_code, enforce=True) + if self.available_languages == '': + return [] + return self.available_languages.split(',') def get_available_languages_as_dicts(self): """ @@ -123,28 +140,32 @@ class App(AbstractApp): last_updated_date = models.DateTimeField(auto_now=True) tracked_remote = models.ForeignKey('RemoteApp', null=True, default=None, on_delete=models.SET_NULL) - translations = TranslatedFields( - feature_graphic=models.ImageField(blank=True, max_length=1024, - upload_to=get_graphic_asset_file_path), - high_res_icon=models.ImageField(blank=True, max_length=1024, - upload_to=get_graphic_asset_file_path), - tv_banner=models.ImageField(blank=True, max_length=1024, - upload_to=get_graphic_asset_file_path), - ) + + # Translated fields + feature_graphic = models.ImageField(blank=True, max_length=1024, + upload_to=get_graphic_asset_file_path) + high_res_icon = models.ImageField(blank=True, max_length=1024, + upload_to=get_graphic_asset_file_path) + tv_banner = models.ImageField(blank=True, max_length=1024, + upload_to=get_graphic_asset_file_path) def get_absolute_url(self): kwargs = {'repo_id': self.repo.pk, 'app_id': self.pk} - try: - kwargs['lang'] = self.language_code - except AttributeError: + + lang = get_language() + if lang in self.get_available_languages(): + kwargs['lang'] = lang + else: kwargs['lang'] = self.get_available_languages()[0] return reverse('app', kwargs=kwargs) def get_edit_url(self): kwargs = {'repo_id': self.repo.pk, 'app_id': self.pk} - try: - kwargs['lang'] = self.language_code - except AttributeError: + + lang = get_language() + if lang in self.get_available_languages(): + kwargs['lang'] = lang + else: kwargs['lang'] = self.get_available_languages()[0] return reverse('app_edit', kwargs=kwargs) @@ -174,21 +195,21 @@ class App(AbstractApp): language_code = to_universal_language_code(original_language_code) if language_code not in localized: localized[language_code] = dict() - app = self.get_translation(original_language_code) - if app.summary: - localized[language_code]['summary'] = app.summary - if app.description: - localized[language_code]['description'] = app.description - if app.feature_graphic: - localized[language_code]['featureGraphic'] = os.path.basename( - app.feature_graphic.name) - if app.high_res_icon: - localized[language_code]['icon'] = os.path.basename(app.high_res_icon.name) - if app.tv_banner: - localized[language_code]['tvBanner'] = os.path.basename(app.tv_banner.name) - if localized[language_code] == {}: - # remove empty translation - del localized[language_code] + with translation.override(original_language_code): + if self.summary: + localized[language_code]['summary'] = self.summary + if self.description: + localized[language_code]['description'] = self.description + if self.feature_graphic: + localized[language_code]['featureGraphic'] = os.path.basename( + self.feature_graphic.name) + if self.high_res_icon: + localized[language_code]['icon'] = os.path.basename(self.high_res_icon.name) + if self.tv_banner: + localized[language_code]['tvBanner'] = os.path.basename(self.tv_banner.name) + if localized[language_code] == {}: + # remove empty translation + del localized[language_code] def _get_screenshot_dict(self): from . import Screenshot @@ -212,24 +233,26 @@ class App(AbstractApp): and ensures that at least one translation exists at the end. """ from .remoteapp import RemoteApp + remote_app = RemoteApp.objects.get(pk=remote_app.pk) for language_code in remote_app.get_available_languages(): - # get the translation for current language_code - remote_app = RemoteApp.objects.language(language_code).get(pk=remote_app.pk) - # copy the translation to this App instance - if language_code in self.get_available_languages(): - app = App.objects.language(language_code).get(pk=self.pk) - app.summary = remote_app.summary - app.description = clean(remote_app.description) - app.save() - else: + summary_field = 'summary_{}'.format(language_code.replace('-', '_')) + description_field = 'description_{}'.format(language_code.replace('-', '_')) + + if language_code not in self.get_available_languages(): self.translate(language_code) - self.summary = remote_app.summary - self.description = clean(remote_app.description) - self.save() + + summary = getattr(remote_app, summary_field) + if summary: + setattr(self, summary_field, summary) + description = getattr(remote_app, description_field) + if description: + setattr(self, description_field, clean(description)) + # ensure that at least one translation exists if len(self.get_available_languages()) == 0: self.default_translate() - self.save() + + self.save() def download_graphic_assets_from_remote_app(self, remote_app): """ @@ -240,34 +263,34 @@ class App(AbstractApp): from .remoteapp import RemoteApp for language_code in remote_app.get_available_languages(): # get the translation for current language_code - app = self.get_translation(language_code) - remote_app = RemoteApp.objects.language(language_code).get(pk=remote_app.pk) - if remote_app.feature_graphic_url: - graphic, etag = net.http_get(remote_app.feature_graphic_url, - remote_app.feature_graphic_etag) - if graphic is not None: - app.feature_graphic.delete() - graphic_name = os.path.basename(remote_app.feature_graphic_url) - app.feature_graphic.save(graphic_name, BytesIO(graphic), save=False) - remote_app.feature_graphic_etag = etag - if remote_app.high_res_icon_url: - graphic, etag = net.http_get(remote_app.high_res_icon_url, - remote_app.high_res_icon_etag) - if graphic is not None: - app.high_res_icon.delete() - graphic_name = os.path.basename(remote_app.high_res_icon_url) - app.high_res_icon.save(graphic_name, BytesIO(graphic), save=False) - remote_app.high_res_icon_etag = etag - if remote_app.tv_banner_url: - graphic, etag = net.http_get(remote_app.tv_banner_url, - remote_app.tv_banner_etag) - if graphic is not None: - app.tv_banner.delete() - graphic_name = os.path.basename(remote_app.tv_banner_url) - app.tv_banner.save(graphic_name, BytesIO(graphic), save=False) - remote_app.tv_banner_etag = etag - app.save() - remote_app.save() + with translation.override(language_code): + remote_app = RemoteApp.objects.get(pk=remote_app.pk) + if remote_app.feature_graphic_url: + graphic, etag = net.http_get(remote_app.feature_graphic_url, + remote_app.feature_graphic_etag) + if graphic is not None: + self.feature_graphic.delete() + graphic_name = os.path.basename(remote_app.feature_graphic_url) + self.feature_graphic.save(graphic_name, BytesIO(graphic), save=False) + remote_app.feature_graphic_etag = etag + if remote_app.high_res_icon_url: + graphic, etag = net.http_get(remote_app.high_res_icon_url, + remote_app.high_res_icon_etag) + if graphic is not None: + self.high_res_icon.delete() + graphic_name = os.path.basename(remote_app.high_res_icon_url) + self.high_res_icon.save(graphic_name, BytesIO(graphic), save=False) + remote_app.high_res_icon_etag = etag + if remote_app.tv_banner_url: + graphic, etag = net.http_get(remote_app.tv_banner_url, + remote_app.tv_banner_etag) + if graphic is not None: + self.tv_banner.delete() + graphic_name = os.path.basename(remote_app.tv_banner_url) + self.tv_banner.save(graphic_name, BytesIO(graphic), save=False) + remote_app.tv_banner_etag = etag + self.save() + remote_app.save() # noinspection PyTypeChecker def update_from_tracked_remote_app(self, remote_apk_pointer): @@ -286,7 +309,7 @@ class App(AbstractApp): if not self.pk: self.save() # save before adding categories, so pk exists - self.category = self.tracked_remote.category.all() + self.category.set(self.tracked_remote.category.all()) self.copy_translations_from_remote_app(self.tracked_remote) diff --git a/repomaker/models/remoteapp.py b/repomaker/models/remoteapp.py index 42d8131..a03e0f6 100644 --- a/repomaker/models/remoteapp.py +++ b/repomaker/models/remoteapp.py @@ -7,10 +7,9 @@ from django.core.exceptions import ObjectDoesNotExist, ValidationError from django.db import models from django.db.models.signals import post_delete from django.dispatch import receiver -from django.utils import timezone +from django.utils import timezone, translation from django.utils.translation import ugettext_lazy as _ from fdroidserver import net -from hvad.models import TranslatedFields from repomaker import tasks from repomaker.tasks import PRIORITY_REMOTE_APP_ICON @@ -24,14 +23,13 @@ class RemoteApp(AbstractApp): repo = models.ForeignKey(RemoteRepository, on_delete=models.CASCADE) icon_etag = models.CharField(max_length=128, blank=True, null=True) last_updated_date = models.DateTimeField(blank=True) - translations = TranslatedFields( - feature_graphic_url=models.URLField(blank=True, max_length=2048), - feature_graphic_etag=models.CharField(max_length=128, blank=True, null=True), - high_res_icon_url=models.URLField(blank=True, max_length=2048), - high_res_icon_etag=models.CharField(max_length=128, blank=True, null=True), - tv_banner_url=models.URLField(blank=True, max_length=2048), - tv_banner_etag=models.CharField(max_length=128, blank=True, null=True), - ) + # Translated fields + feature_graphic_url = models.URLField(blank=True, max_length=2048) + feature_graphic_etag = models.CharField(max_length=128, blank=True, null=True) + high_res_icon_url = models.URLField(blank=True, max_length=2048) + high_res_icon_etag = models.CharField(max_length=128, blank=True, null=True) + tv_banner_url = models.URLField(blank=True, max_length=2048) + tv_banner_etag = models.CharField(max_length=128, blank=True, null=True) def update_from_json(self, app): """ @@ -139,37 +137,34 @@ class RemoteApp(AbstractApp): # TODO also support 'name, 'whatsNew' and 'video' supported_fields = ['summary', 'description', 'featureGraphic', 'icon', 'tvBanner'] available_languages = self.get_available_languages() - for original_language_code, translation in localized.items(): - if set(supported_fields).isdisjoint(translation.keys()): + for original_language_code, app_translation in localized.items(): + if set(supported_fields).isdisjoint(app_translation.keys()): continue # no supported fields in translation # store language code in lower-case, because in Django they are all lower-case as well language_code = original_language_code.lower() # TODO not only add, but also remove old translations again - if language_code in available_languages: - # we need to retrieve the existing translation - app = RemoteApp.objects.language(language_code).get(pk=self.pk) - app.apply_translation(original_language_code, translation) - else: - # create a new translation + if language_code not in available_languages: self.translate(language_code) - self.apply_translation(original_language_code, translation) + + with translation.override(language_code): + self.apply_translation(original_language_code, app_translation) # pylint: disable=attribute-defined-outside-init # noinspection PyAttributeOutsideInit - def apply_translation(self, original_language_code, translation): + def apply_translation(self, original_language_code, new_translation): # textual metadata - if 'summary' in translation: - self.summary = translation['summary'] - if 'description' in translation: - self.description = clean(translation['description']) + if 'summary' in new_translation: + self.summary = new_translation['summary'] + if 'description' in new_translation: + self.description = clean(new_translation['description']) # graphic assets url = self._get_base_url(original_language_code) - if 'featureGraphic' in translation: - self.feature_graphic_url = url + translation['featureGraphic'] - if 'icon' in translation: - self.high_res_icon_url = url + translation['icon'] - if 'tvBanner' in translation: - self.tv_banner_url = url + translation['tvBanner'] + if 'featureGraphic' in new_translation: + self.feature_graphic_url = url + new_translation['featureGraphic'] + if 'icon' in new_translation: + self.high_res_icon_url = url + new_translation['icon'] + if 'tvBanner' in new_translation: + self.tv_banner_url = url + new_translation['tvBanner'] self.save() def _update_screenshots(self, localized): diff --git a/repomaker/models/repository.py b/repomaker/models/repository.py index 2964680..78dfdb3 100644 --- a/repomaker/models/repository.py +++ b/repomaker/models/repository.py @@ -1,3 +1,4 @@ +import fdroidserver import logging import os from io import BytesIO @@ -15,7 +16,7 @@ from django.template.loader import render_to_string from django.templatetags.static import static from django.urls import reverse from django.utils import timezone -from fdroidserver import common, index, server, update +from fdroidserver import common, deploy, update from repomaker import tasks from repomaker.storage import REPO_DIR, get_repo_file_path, get_repo_root_path, \ get_icon_file_path @@ -79,10 +80,10 @@ class AbstractRepository(models.Model): common.fill_config_defaults(config) common.config = config common.options = Options + deploy.config = config + deploy.options = Options update.config = config update.options = Options - server.config = config - server.options = Options return config @@ -118,9 +119,6 @@ class Repository(AbstractRepository): }) if self.icon: config['repo_icon'] = self.icon.name - else: - config['repo_icon'] = os.path.join(settings.BASE_DIR, 'repomaker', 'static', - REPO_DEFAULT_ICON) if self.public_key is not None: config['repo_pubkey'] = self.public_key return config @@ -159,7 +157,8 @@ class Repository(AbstractRepository): # Generate keystore pubkey, fingerprint = common.genkeystore(config) - self.public_key = pubkey + # pubkey is returned as bytes in fdroidserver 2.1.x + self.public_key = pubkey.decode('ascii') self.fingerprint = fingerprint.replace(" ", "") # Generate and save QR Code @@ -337,13 +336,14 @@ class Repository(AbstractRepository): file['packageName'] = pointer.apk.package_id apks.append(file) + update.read_added_date_from_all_apks(apps, apks) update.apply_info_from_latest_apk(apps, apks) # Sort the app list by name sortedids = sorted(apps.keys(), key=lambda app_id: apps[app_id].Name.upper()) # Make the index for the repo - index.make(apps, sortedids, apks, REPO_DIR, False) + fdroidserver.make_index(apps, apks, REPO_DIR, False) update.make_categories_txt(REPO_DIR, categories) # Update cache if it changed @@ -366,7 +366,7 @@ class Repository(AbstractRepository): return # bail out if there is no remote storage to publish to # Publish to remote storage - self.chdir() # expected by server.update_awsbucket() + self.chdir() # expected by update_awsbucket() for storage in remote_storage: storage.publish() diff --git a/repomaker/models/screenshot.py b/repomaker/models/screenshot.py index 6aedad0..ebbc4e3 100644 --- a/repomaker/models/screenshot.py +++ b/repomaker/models/screenshot.py @@ -96,7 +96,7 @@ class RemoteScreenshot(AbstractScreenshot): and creates a local Screenshot if successful. """ screenshot = Screenshot(language_code=self.language_code, type=self.type, app_id=app_id) - r = requests.get(self.url) + r = requests.get(self.url, timeout=60) if r.status_code == requests.codes.ok: screenshot.file.save(os.path.basename(self.url), BytesIO(r.content), save=True) diff --git a/repomaker/models/storage.py b/repomaker/models/storage.py index bf4a22e..6ef6b6d 100644 --- a/repomaker/models/storage.py +++ b/repomaker/models/storage.py @@ -1,4 +1,5 @@ import base64 +import fdroidserver import hashlib import hmac import logging @@ -12,12 +13,12 @@ from cryptography.hazmat.primitives.asymmetric import rsa from django.conf import settings from django.contrib.sites.models import Site from django.core.files.base import ContentFile -from django.core.validators import RegexValidator, ValidationError, slug_re, force_text +from django.core.validators import RegexValidator, ValidationError, slug_re from django.db import models from django.urls import reverse_lazy from django.utils.deconstruct import deconstructible +from django.utils.encoding import force_str from django.utils.translation import ugettext_lazy as _ -from fdroidserver import server from libcloud.storage.types import Provider from repomaker.storage import get_identity_file_path, PrivateStorage, REPO_DIR @@ -92,7 +93,7 @@ class S3Storage(AbstractStorage): config['awsbucket'] = self.bucket config['awsaccesskeyid'] = self.accesskeyid config['awssecretkey'] = self.secretkey - server.update_awsbucket(REPO_DIR) + fdroidserver.update_awsbucket(REPO_DIR) @deconstructible @@ -125,7 +126,7 @@ class HostnameValidator(RegexValidator): message = _('Enter a valid hostname.') def __call__(self, value): - value = force_text(value) + value = force_str(value) # The maximum length of a full host name is 253 characters per RFC 1034 # section 3.1. It's defined to be 255 bytes or less, but this includes # one byte for the length of the name and one byte for the trailing dot @@ -227,7 +228,7 @@ class SshStorage(AbstractSshStorage): super(SshStorage, self).publish() local = self.repo.get_repo_path() remote = self.get_remote_url() - server.update_serverwebroot(remote, local) + fdroidserver.update_serverwebroot(remote, local) class GitStorage(AbstractSshStorage): @@ -252,7 +253,7 @@ class GitStorage(AbstractSshStorage): def publish(self): super(GitStorage, self).publish() remote = [self.get_remote_url()] # a list is expected - server.update_servergitmirrors(remote, REPO_DIR) + fdroidserver.update_servergitmirrors(remote, REPO_DIR) class StorageManager: @@ -347,4 +348,4 @@ class DefaultStorage: remote = os.path.join(self.path, self.get_identifier()) if not os.path.exists(remote): os.makedirs(remote) - server.update_serverwebroot(remote, local) + fdroidserver.update_serverwebroot(remote, local) diff --git a/repomaker/settings.py b/repomaker/settings.py index 0c75456..bd7a49d 100644 --- a/repomaker/settings.py +++ b/repomaker/settings.py @@ -71,7 +71,7 @@ INSTALLED_APPS = [ 'compressor', 'sass_processor', 'background_task', - 'hvad', # model i18n + 'modeltranslation', # model i18n 'tinymce', 'django_js_reverse', 'django.forms', @@ -197,6 +197,11 @@ MAX_ATTEMPTS = 23 # the number of attempts for marking a task as permanently fa LANGUAGE_CODE = 'en' LANGUAGES = [('en-us', ugettext_lazy('American English'))] + global_settings.LANGUAGES +# defaults to the value of LANGUAGES +# for the unit tests to pass, this list need to include at least: +# en, en-us, de, and de-de +MODELTRANSLATION_LANGUAGES = ('en-us', 'en', 'de-de', 'de', 'fr', 'zh-cn') + TIME_ZONE = 'UTC' USE_I18N = True diff --git a/repomaker/settings_local.py b/repomaker/settings_local.py new file mode 100644 index 0000000..16c8865 --- /dev/null +++ b/repomaker/settings_local.py @@ -0,0 +1,35 @@ +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'verbose': { + 'format': '{levelname} {asctime} {module} {process:d} {thread:d} {message}', + 'style': '{', + }, + 'simple': { + 'format': '{levelname} {message}', + 'style': '{', + }, + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + }, + }, + 'loggers': { + 'root': { + 'handlers': ['console'], + 'formatter': 'verbose', + 'level': 'INFO', + }, + # 'django': { + # 'handlers': ['console'], + # 'level': 'DEBUG', + # 'propagate': True, + # }, + 'repomaker': { + 'level': 'DEBUG', + 'propagate': True, + }, + }, +} diff --git a/repomaker/settings_test.py b/repomaker/settings_test.py index bf7464a..18e674d 100644 --- a/repomaker/settings_test.py +++ b/repomaker/settings_test.py @@ -1,9 +1,10 @@ +import tempfile from repomaker.settings import * # pylint: disable=wildcard-import,unused-wildcard-import TEST_FILES_DIR = os.path.join(BASE_DIR, 'tests') -TEST_DIR = os.path.join(BASE_DIR, 'test_dir') +TEST_DIR = os.path.join(tempfile.gettempdir(), 'test_dir') MEDIA_ROOT = os.path.join(TEST_DIR, 'media') PRIVATE_REPO_ROOT = os.path.join(TEST_DIR, 'private_repo') STATIC_ROOT = os.path.join(TEST_DIR, 'static') diff --git a/repomaker/storage.py b/repomaker/storage.py index 6448eb6..a1974a6 100644 --- a/repomaker/storage.py +++ b/repomaker/storage.py @@ -3,6 +3,7 @@ import re from django.conf import settings from django.core.files.storage import FileSystemStorage +from modeltranslation.utils import get_language from repomaker.utils import to_universal_language_code REPO_DIR = 'repo' @@ -37,9 +38,8 @@ def get_apk_file_path(apk, filename): return os.path.join('packages', filename) -def get_graphic_asset_file_path(app_translation, filename): - app = app_translation.master - language_code = to_universal_language_code(app_translation.language_code) +def get_graphic_asset_file_path(app, filename): + language_code = to_universal_language_code(get_language()) path = os.path.join(get_repo_path(app.repo), app.package_id, language_code) return os.path.join(path, filename) diff --git a/repomaker/templates/repomaker/app/remote_add.html b/repomaker/templates/repomaker/app/remote_add.html index 326caac..5dd8d06 100644 --- a/repomaker/templates/repomaker/app/remote_add.html +++ b/repomaker/templates/repomaker/app/remote_add.html @@ -38,7 +38,7 @@ {% trans 'There are currently no screenshots shown because this gives the repo owner the ability to track you.' %}
- +
diff --git a/repomaker/templates/repomaker/repo_page/index.html b/repomaker/templates/repomaker/repo_page/index.html index 367d682..cee7209 100644 --- a/repomaker/templates/repomaker/repo_page/index.html +++ b/repomaker/templates/repomaker/repo_page/index.html @@ -56,7 +56,7 @@

{{ repo.name }}

{{ repo.description }}

- {% for app in repo.app_set.language.fallbacks.all %} + {% for app in repo.app_set.all %} {% include "repomaker/widgets/app.html" with repo_page=True width=6 no_hover=True %} {% endfor %} diff --git a/repomaker/tests/models/test_apk.py b/repomaker/tests/models/test_apk.py index 3d1633d..29f3043 100644 --- a/repomaker/tests/models/test_apk.py +++ b/repomaker/tests/models/test_apk.py @@ -87,7 +87,7 @@ class ApkTestCase(RmTestCase): # download file and assert there was a GET request for the URL self.apk.download('url/download.apk') - get.assert_called_once_with('url/download.apk') + get.assert_called_once_with('url/download.apk', timeout=60) # assert that downloaded file has been saved self.assertEqual(get_apk_file_path(self.apk, 'download.apk'), self.apk.file.name) @@ -194,7 +194,7 @@ class ApkTestCase(RmTestCase): # download file and assert there was a GET request for the URL self.apk.download('url/test.mp4') - get.assert_called_once_with('url/test.mp4') + get.assert_called_once_with('url/test.mp4', timeout=60) # assert that downloaded file has been saved self.assertEqual(get_apk_file_path(self.apk, 'test.mp4'), self.apk.file.name) @@ -225,12 +225,6 @@ class ApkTestCase(RmTestCase): self.assertTrue(datetime_is_recent(apk.added_date)) self.assertFalse(apk.is_downloading) - def test_initialize_rejects_md5_apk(self): - with open(os.path.join(settings.TEST_FILES_DIR, 'test_md5_signature.apk'), 'rb') as f: - self.apk.file.save('test_md5_signature.apk', f, save=True) - with self.assertRaises(ValidationError): - self.apk.initialize() - def test_initialize_rejects_invalid_apk(self): # overwrite APK file with rubbish self.apk.file.delete() @@ -238,7 +232,7 @@ class ApkTestCase(RmTestCase): with self.assertRaises(ValidationError): self.apk.initialize() - @patch('fdroidserver.update.scan_apk') + @patch('fdroidserver.scan_apk') def test_initialize_rejects_invalid_apk_scan(self, scan_apk): scan_apk.side_effect = BuildException with self.assertRaises(ValidationError): diff --git a/repomaker/tests/models/test_app.py b/repomaker/tests/models/test_app.py index 4b4ca88..3c94c8f 100644 --- a/repomaker/tests/models/test_app.py +++ b/repomaker/tests/models/test_app.py @@ -6,6 +6,7 @@ import os from background_task.models import Task from django.conf import settings from django.core.files.base import ContentFile, File +from django.utils import translation from repomaker.models import RemoteRepository, App, Apk, RemoteApkPointer, RemoteApp, Screenshot, \ ApkPointer, Category @@ -36,26 +37,28 @@ class AppTestCase(RmTestCase): # add two translations to RemoteApp remote_app.translate('en-us') - remote_app.summary = 'dog' - remote_app.description = 'cat' - remote_app.save() + with translation.override('en-us'): + remote_app.summary = 'dog' + remote_app.description = 'cat' remote_app.translate('de') - remote_app.summary = 'hund' - remote_app.description = 'katze' + with translation.override('de'): + remote_app.summary = 'hund' + remote_app.description = 'katze' remote_app.save() # copy the translations to the App app.copy_translations_from_remote_app(remote_app) + app = App.objects.get(pk=app.pk) # assert that English translation was copied - app = App.objects.language('en-us').get(pk=app.pk) - self.assertEqual('dog', app.summary) - self.assertEqual('cat', app.description) + with translation.override('en-us'): + self.assertEqual('dog', app.summary) + self.assertEqual('cat', app.description) # assert that German translation was copied - app = App.objects.language('de').get(pk=app.pk) - self.assertEqual('hund', app.summary) - self.assertEqual('katze', app.description) + with translation.override('de'): + self.assertEqual('hund', app.summary) + self.assertEqual('katze', app.description) def test_copy_translations_from_remote_app_default_translation(self): # copy the non-existent translations to the App @@ -99,15 +102,16 @@ class AppTestCase(RmTestCase): self.assertEqual({'en-us', 'de'}, set(self.app.get_available_languages())) # add also graphic assets - app = App.objects.language('de').get(pk=self.app.pk) - app.feature_graphic.save('feature.png', io.BytesIO(b'foo'), save=False) - app.high_res_icon.save('icon.png', io.BytesIO(b'foo'), save=False) - app.tv_banner.save('tv.png', io.BytesIO(b'foo'), save=False) - app.save() + with translation.override('de'): + app = App.objects.get(pk=self.app.pk) + app.feature_graphic.save('feature.png', io.BytesIO(b'foo'), save=False) + app.high_res_icon.save('icon.png', io.BytesIO(b'foo'), save=False) + app.tv_banner.save('tv.png', io.BytesIO(b'foo'), save=False) + app.save() # get localized dict localized = {'en-US': {'otherKey': 'test'}} - # noinspection PyProtectedMember + app._add_translations_to_localized(localized) # pylint: disable=protected-access # assert that dict was created properly @@ -142,31 +146,32 @@ class AppTestCase(RmTestCase): # set initial feature graphic for app app.translate('de') app.save() # needs to be saved for ForeignKey App to be available when saving file - app.feature_graphic.save('feature.png', io.BytesIO(b'foo'), save=True) - old_feature_graphic_path = app.feature_graphic.path - self.assertTrue(os.path.isfile(old_feature_graphic_path)) + with translation.override('de'): + app.feature_graphic.save('feature.png', io.BytesIO(b'foo'), save=True) + old_feature_graphic_path = app.feature_graphic.path + self.assertTrue(os.path.isfile(old_feature_graphic_path)) - # add graphics to remote app - remote_app.translate('de') - remote_app.feature_graphic_url = 'http://url/feature-graphic.png' - remote_app.feature_graphic_etag = 'etag' - remote_app.save() + remote_app.translate('de') - # download graphic assets - http_get.return_value = b'icon-data', 'new_etag' - app.download_graphic_assets_from_remote_app(remote_app) - http_get.assert_called_once_with(remote_app.feature_graphic_url, 'etag') + remote_app.feature_graphic_url = 'http://url/feature-graphic.png' + remote_app.feature_graphic_etag = 'etag' + remote_app.save() - # assert that old feature graphic got deleted and new one was saved - app = App.objects.language('de').get(pk=app.pk) - self.assertFalse(os.path.isfile(old_feature_graphic_path)) - self.assertEqual('user_1/repo_1/repo/org.example/de/feature-graphic.png', - app.feature_graphic.name) - self.assertTrue(os.path.isfile(app.feature_graphic.path)) + # download graphic assets + http_get.return_value = b'icon-data', 'new_etag' + app.download_graphic_assets_from_remote_app(remote_app) + http_get.assert_called_once_with(remote_app.feature_graphic_url, 'etag') - # assert that new etag was saved - remote_app = RemoteApp.objects.language('de').get(pk=remote_app.pk) - self.assertEqual('new_etag', remote_app.feature_graphic_etag) + # assert that old feature graphic got deleted and new one was saved + app = App.objects.get(pk=app.pk) + self.assertFalse(os.path.isfile(old_feature_graphic_path)) + self.assertEqual('user_1/repo_1/repo/org.example/de/feature-graphic.png', + app.feature_graphic.name) + self.assertTrue(os.path.isfile(app.feature_graphic.path)) + + # assert that new etag was saved + remote_app = RemoteApp.objects.get(pk=remote_app.pk) + self.assertEqual('new_etag', remote_app.feature_graphic_etag) @patch('repomaker.models.app.App.copy_translations_from_remote_app') @patch('repomaker.models.app.App.add_apk_from_tracked_remote_app') @@ -270,8 +275,8 @@ class AppTestCase(RmTestCase): self.app.update_icon(new_icon) # assert that new icon has been saved properly - with open(self.app.icon.path, 'r') as f1: - with open(self.remote_app.icon.path, 'r') as f2: + with open(self.app.icon.path, 'rb') as f1: + with open(self.remote_app.icon.path, 'rb') as f2: self.assertEqual(f1.read(), f2.read()) self.assertTrue(self.app.icon.name.endswith('test2.png')) diff --git a/repomaker/tests/models/test_remoteapp.py b/repomaker/tests/models/test_remoteapp.py index 6e1beb2..f366d83 100644 --- a/repomaker/tests/models/test_remoteapp.py +++ b/repomaker/tests/models/test_remoteapp.py @@ -6,7 +6,9 @@ import os from django.conf import settings from django.contrib.auth.models import User from django.core.exceptions import ValidationError +from django.utils import translation +from repomaker import DEFAULT_USER_NAME from repomaker.models import Repository, RemoteRepository, App, RemoteApp, Apk, ApkPointer, \ RemoteApkPointer, RemoteScreenshot from repomaker.models.screenshot import PHONE @@ -18,13 +20,24 @@ from .. import datetime_is_recent, RmTestCase class RemoteAppTestCase(RmTestCase): repo = None app = None + local_repo = None def setUp(self): + if not settings.SINGLE_USER_MODE: + self.user = User.objects.create(username=DEFAULT_USER_NAME) + self.client.force_login(user=self.user) + else: + self.user = User.objects.get() + date = datetime.fromtimestamp(1337, timezone.utc) self.repo = RemoteRepository.objects.create(name='Test', url='http://repo_url', last_change_date=date) self.app = RemoteApp.objects.create(repo=self.repo, package_id="org.example", last_updated_date=date) + self.local_repo = Repository.objects.create(name='Test Local', + url='http://repo_local', + fingerprint="local_fp", + user=self.user) def test_update_from_json_only_when_update(self): json = {'name': 'app', 'lastUpdated': 10000} @@ -86,7 +99,8 @@ class RemoteAppTestCase(RmTestCase): self.assertTrue(os.path.isfile(old_icon_path)) # create one local app tracking the remote one - App.objects.create(repo_id=1, package_id=self.app.package_id, tracked_remote=self.app) + App.objects.create(repo_id=self.local_repo.pk, package_id=self.app.package_id, + tracked_remote=self.app) # update icon http_get.return_value = b'icon-data', 'new_etag' @@ -114,61 +128,66 @@ class RemoteAppTestCase(RmTestCase): self.app._update_translations(localized) # pylint: disable=protected-access # assert that translation has been saved - app = RemoteApp.objects.language('en').get(pk=self.app.pk) - self.assertEqual(localized['en']['summary'], app.summary) - self.assertEqual(localized['en']['description'], app.description) + with translation.override('en'): + app = RemoteApp.objects.get(pk=self.app.pk) + self.assertEqual(localized['en']['summary'], app.summary) + self.assertEqual(localized['en']['description'], app.description) def test_update_translations_existing(self): # add a new translation self.test_update_translations_new() - self.assertTrue(RemoteApp.objects.language('en').exists()) + self.assertTrue('en' in self.app.get_available_languages()) + # self.assertTrue(RemoteApp.objects.language('en').exists()) # update existing translation localized = {'en': {'summary': 'newfoo', 'description': 'newbar', 'video': 'bla'}} self.app._update_translations(localized) # pylint: disable=protected-access # assert that translation has been updated - app = RemoteApp.objects.language('en').get(pk=self.app.pk) - self.assertEqual(localized['en']['summary'], app.summary) - self.assertEqual(localized['en']['description'], app.description) + with translation.override('en'): + self.assertEqual(localized['en']['summary'], self.app.summary) + self.assertEqual(localized['en']['description'], self.app.description) def test_update_translations_lowercase_language_code(self): # update remote app translation with a new one localized = {'en-US': {'summary': 'foo', 'description': 'bar', 'featureGraphic': 'test'}} self.app._update_translations(localized) # pylint: disable=protected-access - # assert that translation has been saved with an all lower-case language code - app = RemoteApp.objects.language('en-us').get(pk=self.app.pk) - self.assertEqual(localized['en-US']['summary'], app.summary) - self.assertEqual(localized['en-US']['description'], app.description) + with translation.override('en'): + # assert that translation has been saved with an all lower-case language code + self.assertEqual(localized['en-US']['summary'], self.app.summary) + self.assertEqual(localized['en-US']['description'], self.app.description) - # assert that language_code in URL was not changed - self.assertEqual('http://repo_url/org.example/en-US/test', app.feature_graphic_url) + # assert that language_code in URL was not changed + self.assertEqual('http://repo_url/org.example/en-US/test', self.app.feature_graphic_url) def test_apply_translation(self): # apply new translation - translation = {'summary': 'test1', 'description': 'test2', 'featureGraphic': 'feature.png', - 'icon': 'icon.png', 'tvBanner': 'tv.png'} + new_translation = {'summary': 'test1', 'description': 'test2', + 'featureGraphic': 'feature.png', 'icon': 'icon.png', + 'tvBanner': 'tv.png'} self.app.translate('de') - self.app.apply_translation('de', translation) + with translation.override('de'): + self.app.apply_translation('de', new_translation) - # assert that translation has been saved - app = RemoteApp.objects.language('de').get(pk=self.app.pk) - self.assertEqual(translation['summary'], app.summary) - self.assertEqual(translation['description'], app.description) - self.assertEqual('http://repo_url/org.example/de/feature.png', app.feature_graphic_url) - self.assertEqual('http://repo_url/org.example/de/icon.png', app.high_res_icon_url) - self.assertEqual('http://repo_url/org.example/de/tv.png', app.tv_banner_url) + # assert that translation has been saved + self.assertEqual(new_translation['summary'], self.app.summary) + self.assertEqual(new_translation['description'], self.app.description) + self.assertEqual('http://repo_url/org.example/de/feature.png', + self.app.feature_graphic_url) + self.assertEqual('http://repo_url/org.example/de/icon.png', self.app.high_res_icon_url) + self.assertEqual('http://repo_url/org.example/de/tv.png', self.app.tv_banner_url) def test_apply_translation_sanitation(self): # apply new translation - translation = {'summary': 'foo', 'description': 'test2