Add pre-release script and move stuff to data directory

.gitignore

@@ -1,14 +1,14 @@
-db.sqlite3
+.coverage
 .idea
 *.pyc
-media/user_*
-media/remote_repo_*
-media/packages
-private_repo
 credentials
+data/db.sqlite3
+data/media/packages/
+data/media/remote_repo_*
+data/media/user_*
+data/private_repo/
+data/static/
 node_modules/
-/static/
-.coverage
 repomaker/static/repomaker/css/styles.css
 repomaker/static/repomaker/css/repo/page.css
 

pre-release.sh

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+set -x
+shopt -s extglob
+
+NODE=node_modules
+
+# compile translations
+python3 manage.py compilemessages
+
+# update npm packages
+npm install
+
+# material-design-icons-iconfont remove unused files
+rm -rf ${NODE}/material-design-icons-iconfont/!(dist)
+rm -rf ${NODE}/material-design-icons-iconfont/.idea
+
+# material-design-lite remove unused files
+rm -rf ${NODE}/material-design-lite/!(dist|src|material.min.js)
+rm -rf ${NODE}/material-design-lite/dist/!(images)
+rm -rf ${NODE}/material-design-lite/.[tj]*
+
+# roboto-fontface remove unused files
+rm -rf ${NODE}/roboto-fontface/!(css|fonts)
+rm -rf ${NODE}/roboto-fontface/.npmignore
+
+# tinymce remove unused files
+rm -rf ${NODE}/tinymce/!(plugins|skins|themes|tinymce.min.js|tinymce.js)
+rm -rf ${NODE}/tinymce/themes/!(modern)
+rm -rf ${NODE}/tinymce/plugins/!(autolink|link|lists)
+
+# compile stylesheets
+python3 manage.py compilescss
+
+# rename folders so they can be ignored
+mv ${NODE}/material-design-lite/src ${NODE}/material-design-lite/src-ignore
+mv ${NODE}/roboto-fontface/css ${NODE}/roboto-fontface/css-ignore
+
+# copy static files into STATIC_DIR
+python3 manage.py collectstatic --no-input \
+    -i *.scss -i *.less \
+    -i tiny_mce \
+    -i src-ignore \
+    -i css-ignore \
+    --clear
+
+# rename folders back
+mv ${NODE}/material-design-lite/src-ignore ${NODE}/material-design-lite/src
+mv ${NODE}/roboto-fontface/css-ignore ${NODE}/roboto-fontface/css

purge.sh

@@ -1,17 +1,17 @@
 #!/usr/bin/env bash
 
 # remove local files
-rm -r  private_repo/user_*
-rm -rf media/user_*
-rm -r  media/remote_repo_*
-rm -r  media/packages
+rm -rf data/private_repo/user_*
+rm -rf data/media/user_*
+rm -rf data/media/remote_repo_*
+rm -rf data/media/packages
 
 # remove database and move custom migrations out of the way
 rm repomaker/migrations/0*
 mv repomaker/migrations/default_user.py repomaker/migrations/default_user.py.bak
 mv repomaker/migrations/default_categories.py repomaker/migrations/default_categories.py.bak
 mv repomaker/migrations/default_remote_repositories.py repomaker/migrations/default_remote_repositories.py.bak
-rm db.sqlite3
+rm data/db.sqlite3
 
 # initialize database and re-add custom migrations
 python3 manage.py makemigrations repomaker
@@ -19,4 +19,3 @@ mv repomaker/migrations/default_user.py.bak repomaker/migrations/default_user.py
 mv repomaker/migrations/default_categories.py.bak repomaker/migrations/default_categories.py
 mv repomaker/migrations/default_remote_repositories.py.bak repomaker/migrations/default_remote_repositories.py
 python3 manage.py migrate
-

repomaker/settings.py

@@ -17,6 +17,9 @@ from django.utils.translation import ugettext_lazy
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
+# Where user data such as repositories will be stored
+DATA_DIR = os.path.join(BASE_DIR, 'data')
+
 SINGLE_USER_MODE = True
 
 # Quick-start development settings - unsuitable for production
@@ -33,11 +36,11 @@ logging.getLogger().setLevel(logging.DEBUG)
 ALLOWED_HOSTS = ['127.0.0.1']
 
 # Location for media accessible via the web-server such as repo icons, screenshots, etc.
-MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
+MEDIA_ROOT = os.path.join(DATA_DIR, 'media')
 MEDIA_URL = "/media/"
 
 # Location for private data such as the repo signing key
-PRIVATE_REPO_ROOT = os.path.join(BASE_DIR, 'private_repo')
+PRIVATE_REPO_ROOT = os.path.join(DATA_DIR, 'private_repo')
 
 # Uncomment the next line to show this notice on every page
 # SITE_NOTICE = '''Maintenance ongoing. Please check back later.'''
@@ -48,7 +51,7 @@ PRIVATE_REPO_ROOT = os.path.join(BASE_DIR, 'private_repo')
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+        'NAME': os.path.join(DATA_DIR, 'db.sqlite3'),
     }
 }
 
@@ -56,7 +59,7 @@ DATABASES = {
 # Uncomment and edit this, if you want to offer your users storage for their repositories
 # You need to configure your web-server to serve from those locations
 # DEFAULT_REPO_STORAGE = [
-#     (os.path.join(BASE_DIR, 'repos'), '/repos/'),
+#     (os.path.join(DATA_DIR, 'repos'), '/repos/'),
 #     ('/var/repomaker/repos', 'https://repos.example.org/'),
 # ]
 
@@ -201,10 +204,11 @@ LOCALE_PATHS = (
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/dev/howto/static-files/
 
+STATIC_ROOT = os.path.join(DATA_DIR, 'static')
 STATIC_URL = '/static/'
-STATIC_ROOT = os.path.join(BASE_DIR, 'static')
 
 NODE_MODULES_ROOT = os.path.join(BASE_DIR, 'node_modules')
+NODE_MODULES_URL = STATIC_URL + 'node_modules/'
 
 STATICFILES_DIRS = [
     ('node_modules', NODE_MODULES_ROOT),
@@ -220,8 +224,6 @@ SASS_PROCESSOR_INCLUDE_DIRS = [
     NODE_MODULES_ROOT,
 ]
 
-NODE_MODULES_URL = STATIC_URL + 'node_modules/'
-
 TINYMCE_JS_URL = os.path.join(NODE_MODULES_URL, "tinymce/tinymce.js")
 TINYMCE_DEFAULT_CONFIG = {
     'theme': 'modern',

repomaker/static/repomaker/css/variables.scss

@@ -1,6 +1,6 @@
 // Paths
 $material-design-icons-font-path: unquote(get-setting(NODE_MODULES_URL) + 'material-design-icons-iconfont/dist/fonts/');
-$roboto-font-path: unquote(get-setting(NODE_MODULES_URL) + "roboto-fontface/fonts");
+$roboto-font-path: unquote(get-setting(NODE_MODULES_URL) + 'roboto-fontface/fonts');
 $image_path: unquote(get-setting(NODE_MODULES_URL) + 'material-design-lite/dist/images/');
 $socialaccount_provider_url: unquote(get-setting(STATIC_URL) + 'repomaker/images/socialaccount_providers/');
 

repomaker/storage.py

@@ -3,6 +3,7 @@ import re
 
 from django.conf import settings
 from django.core.files.storage import FileSystemStorage
+from django.contrib.staticfiles.finders import FileSystemFinder
 
 from repomaker.utils import to_universal_language_code
 

setup.py

@@ -2,17 +2,21 @@ import os
 
 from setuptools import setup, find_packages
 
+DATA_PREFIX = os.path.join('share', 'repomaker')
+
 packages = find_packages(exclude=['*.tests*'])
 print("Packages: %s" % str(packages))
 
 
-def all_files_in(directory):
-    files = []
+def get_data_files_in(directory):
+    pairs = []
     for (rel_path, directories, filenames) in os.walk(directory):
+        files = []
         for filename in filenames:
-            # TODO do not store files outside package directory
-            files.append(os.path.join('..', rel_path, filename))
-    return files
+            files.append(os.path.join(rel_path, filename))
+        if len(files) > 0:
+            pairs.append((os.path.join(DATA_PREFIX, rel_path), files))
+    return pairs
 
 
 setup(
@@ -44,9 +48,8 @@ setup(
         'git+https://gitlab.com/fdroid/fdroidserver.git@0be224b3e0c7b82b36bdd3c6bca07e0e6cb4a023#egg=fdroidserver',
     ],
 
-    # List additional groups of dependencies here (e.g. development
-    # dependencies). You can install these using the following syntax,
-    # for example:
+    # List additional groups of dependencies here (e.g. development dependencies).
+    # You can install these using the following syntax, for example:
     # $ pip install -e .[dev,test]
     extras_require={
         'test': [
@@ -56,21 +59,18 @@ setup(
         ],
     },
 
-    # If there are data files included in your packages that need to be
-    # installed, specify them here.  If using Python 2.6 or less, then these
-    # have to be included in MANIFEST.in as well.
-    package_data={
-        'repomaker': [
-            '../locale/*/LC_MESSAGES/*.mo',
-            '../media/default-*.png',
-        ] + all_files_in('node_modules')  # TODO only include what we need
-    },
+    # package_data={},
 
     # Although 'package_data' is the preferred approach, in some case you may
     # need to place data files outside of your packages. See:
-    # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files # noqa
+    # http://docs.python.org/3.5/distutils/setupscript.html#installing-additional-files # noqa
     # In this case, 'data_file' will be installed into '<sys.prefix>/my_data'
-#    data_files=[('my_data', ['data/data_file'])],
+    data_files=get_data_files_in('data/static/') + [
+        (os.path.join(DATA_PREFIX, 'data', 'media'), [
+            'data/media/default-app-icon.png',
+            'data/media/default-repo-icon.png',
+        ])
+    ],
 
     # To provide executable scripts, use entry points in preference to the
     # "scripts" keyword. Entry points provide cross-platform support and allow

setup.sh

@@ -1,7 +1,8 @@
 #!/usr/bin/env bash
 set -x
-pip3 install -r requirements.txt --user && \
-python3 manage.py makemigrations repomaker && \
-python3 manage.py migrate && \
+set -e
+pip3 install -r requirements.txt --user
+mkdir -p data
+python3 manage.py makemigrations repomaker
+python3 manage.py migrate
 echo "All set up, now execute run.sh"
-

update-translations.sh

@@ -2,7 +2,6 @@
 
 python3 manage.py makemessages --keep-pot --no-wrap --no-location --no-obsolete --ignore node_modules -v 3
 python3 manage.py makemessages --keep-pot --no-wrap --no-location --no-obsolete --ignore node_modules -v 3 -d djangojs
-python3 manage.py compilemessages
 
 sed -i -e '/^"POT-Creation-Date: /d' repomaker/locale/*/LC_MESSAGES/django.po
 sed -i -e '/^"POT-Creation-Date: /d' repomaker/locale/*/LC_MESSAGES/djangojs.po