Merge branch 'issue_433' into 'master'

[many] Allow backup to operate on implicit order (c.f. #433)



See merge request !210

beat/web/algorithms/migrations/0003_auto_20160701_1707.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2016-07-01 17:07
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('algorithms', '0002_auto_20160630_1644'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='algorithm',
+            options={'ordering': ['version', 'id']},
+        ),
+    ]

beat/web/common/models.py

@@ -383,7 +383,7 @@ class Versionable(Shareable):
         abstract = True
 
         # setup ordering so that the dump order respects self dependencies
-        ordering = ['previous_version_id', 'fork_of_id', 'id']
+        ordering = ['version', 'id']
 
 
     #_____ Static Methods __________

beat/web/databases/migrations/0004_auto_20160701_1707.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2016-07-01 17:07
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('databases', '0003_auto_20160630_1710'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='database',
+            options={'ordering': ['version', 'id']},
+        ),
+    ]

beat/web/databases/models.py

@@ -195,7 +195,7 @@ class Database(Versionable):
         # setup ordering so that the dump order respects self dependencies
         # note: Versionable already has the right ordering - not sure why
         # `django makemigrations' is not detecting it though.
-        ordering = ['previous_version_id', 'fork_of_id', 'id']
+        ordering = ['version', 'id']
 
 
     #_____ Utilities __________

beat/web/dataformats/migrations/0002_auto_20160701_1712.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2016-07-01 17:12
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dataformats', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='dataformat',
+            options={'ordering': ['version', 'id'], 'verbose_name': 'dataformat', 'verbose_name_plural': 'dataformats'},
+        ),
+        migrations.AlterUniqueTogether(
+            name='dataformat',
+            unique_together=set([('author', 'name', 'version')]),
+        ),
+    ]

beat/web/dataformats/models.py

@@ -221,7 +221,7 @@ class DataFormat(StoredContribution):
 
     #_____ Meta parameters __________
 
-    class Meta:
+    class Meta(StoredContribution.Meta):
         verbose_name = 'dataformat'
         verbose_name_plural = 'dataformats'
 

beat/web/experiments/admin.py

@@ -120,8 +120,9 @@ class BlockInline(admin.TabularInline):
     model = BlockModel
     extra = 0
 
-    readonly_fields = ['link', 'algorithm', 'analyzer', 'status']
-    ordering = ['id']
+    readonly_fields = ['execution_order', 'link', 'algorithm', 'analyzer',
+        'status']
+    ordering = ['execution_order']
     fields = readonly_fields
 
     def link(self, obj):
@@ -299,10 +300,14 @@ class BlockDependentsInline(admin.TabularInline):
     fk_name = 'to_block'
     extra = 0
 
-    readonly_fields = ['name', 'algorithm', 'analyzer', 'status']
+    readonly_fields = ['order', 'name', 'algorithm', 'analyzer',
+        'status']
     ordering = ['id']
     fields = readonly_fields
 
+    def order(self, obj):
+        return obj.from_block.execution_order
+
     def name(self, obj):
         url = reverse('admin:experiments_block_change', args=(obj.from_block.pk,))
         return mark_safe('<a href="%s">%s</a>' % (url, obj.from_block.name))
@@ -332,10 +337,13 @@ class BlockDependenciesInline(admin.TabularInline):
     fk_name = 'from_block'
     extra = 0
 
-    readonly_fields = ['name', 'algorithm', 'analyzer', 'status']
+    readonly_fields = ['order', 'name', 'algorithm', 'analyzer', 'status']
     ordering = ['id']
     fields = readonly_fields
 
+    def order(self, obj):
+        return obj.to_block.execution_order
+
     def name(self, obj):
         url = reverse('admin:experiments_block_change', args=(obj.to_block.pk,))
         return mark_safe('<a href="%s">%s</a>' % (url, obj.to_block.name))
@@ -371,7 +379,7 @@ class BlockModelForm(forms.ModelForm):
 
 class Block(admin.ModelAdmin):
 
-    list_display        = ('id', 'author', 'toolchain', 'xp', 'name', 'algorithm', 'analyzer', 'status', 'ins', 'outs', 'environment', 'q')
+    list_display        = ('id', 'author', 'toolchain', 'xp', 'execution_order', 'name', 'algorithm', 'analyzer', 'status', 'ins', 'outs', 'environment', 'q')
     search_fields       = ['name',
                            'experiment__author__username',
                            'experiment__toolchain__author__username',

beat/web/libraries/migrations/0003_auto_20160701_1707.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2016-07-01 17:07
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('libraries', '0002_auto_20160630_1644'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='library',
+            options={'ordering': ['version', 'id'], 'verbose_name_plural': 'libraries'},
+        ),
+    ]

beat/web/plotters/migrations/0003_auto_20160701_1707.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2016-07-01 17:07
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('plotters', '0002_auto_20160630_1644'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='plotter',
+            options={'ordering': ['version', 'id']},
+        ),
+        migrations.AlterModelOptions(
+            name='plotterparameter',
+            options={'ordering': ['version', 'id']},
+        ),
+    ]

beat/web/toolchains/migrations/0003_auto_20160701_1707.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.4 on 2016-07-01 17:07
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('toolchains', '0002_auto_20160630_1644'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='toolchain',
+            options={'ordering': ['version', 'id']},
+        ),
+    ]

beat/web/utils/management/commands/backup.py

@@ -149,7 +149,7 @@ class Command(BaseCommand):
             destfile = os.path.join(tmpdir, 'initial.json')
             logger.info("Dumping initial (unspecified) data -> `%s'", destfile)
             arguments['output'] = destfile #new in Django-1.8.x
-            call_command('dumpdata', **arguments)
+            call_command('beatdump', **arguments)
 
             # and backs-up the apps respecting the imposed order
             for app in use_apps:
@@ -159,7 +159,7 @@ class Command(BaseCommand):
                 logger.info("Dumping data for `%s' -> `%s'", app, destfile)
 
                 arguments['output'] = destfile #new in Django-1.8.x
-                call_command('dumpdata', only.get(app, app), **arguments)
+                call_command('beatdump', only.get(app, app), **arguments)
 
                 # copy prefix data
                 path = os.path.join(settings.PREFIX, app)

beat/web/utils/management/commands/beatdump.py

+# This file the Django/GitHub equivalent of
+# django/core/management/commands/dumpdata.py. It was copied here so that we
+# can remove the ordering enforcement on the command-level. If you find this
+# file is outdated, please re-download the appropriate Django version of this
+# file and patch it up again. The patch area, which is quite small, is clearly
+# indicated below.
+#
+# Django (BSD License)
+
+from collections import OrderedDict
+
+from django.apps import apps
+from django.core import serializers
+from django.core.management.base import BaseCommand, CommandError
+from django.db import DEFAULT_DB_ALIAS, router
+
+
+class Command(BaseCommand):
+    help = ("Output the contents of the database as a fixture of the given "
+            "format (using each model's default manager unless --all is "
+            "specified).")
+
+    def add_arguments(self, parser):
+        parser.add_argument('args', metavar='app_label[.ModelName]', nargs='*',
+            help='Restricts dumped data to the specified app_label or app_label.ModelName.')
+        parser.add_argument('--format', default='json', dest='format',
+            help='Specifies the output serialization format for fixtures.')
+        parser.add_argument('--indent', default=None, dest='indent', type=int,
+            help='Specifies the indent level to use when pretty-printing output.')
+        parser.add_argument('--database', action='store', dest='database',
+            default=DEFAULT_DB_ALIAS,
+            help='Nominates a specific database to dump fixtures from. '
+                 'Defaults to the "default" database.')
+        parser.add_argument('-e', '--exclude', dest='exclude', action='append', default=[],
+            help='An app_label or app_label.ModelName to exclude '
+                 '(use multiple --exclude to exclude multiple apps/models).')
+        parser.add_argument('--natural-foreign', action='store_true', dest='use_natural_foreign_keys', default=False,
+            help='Use natural foreign keys if they are available.')
+        parser.add_argument('--natural-primary', action='store_true', dest='use_natural_primary_keys', default=False,
+            help='Use natural primary keys if they are available.')
+        parser.add_argument('-a', '--all', action='store_true', dest='use_base_manager', default=False,
+            help="Use Django's base manager to dump all models stored in the database, "
+                 "including those that would otherwise be filtered or modified by a custom manager.")
+        parser.add_argument('--pks', dest='primary_keys',
+            help="Only dump objects with given primary keys. "
+                 "Accepts a comma separated list of keys. "
+                 "This option will only work when you specify one model.")
+        parser.add_argument('-o', '--output', default=None, dest='output',
+            help='Specifies file to which the output is written.')
+
+    def handle(self, *app_labels, **options):
+        format = options.get('format')
+        indent = options.get('indent')
+        using = options.get('database')
+        excludes = options.get('exclude')
+        output = options.get('output')
+        show_traceback = options.get('traceback')
+        use_natural_foreign_keys = options.get('use_natural_foreign_keys')
+        use_natural_primary_keys = options.get('use_natural_primary_keys')
+        use_base_manager = options.get('use_base_manager')
+        pks = options.get('primary_keys')
+
+        if pks:
+            primary_keys = pks.split(',')
+        else:
+            primary_keys = []
+
+        excluded_apps = set()
+        excluded_models = set()
+        for exclude in excludes:
+            if '.' in exclude:
+                try:
+                    model = apps.get_model(exclude)
+                except LookupError:
+                    raise CommandError('Unknown model in excludes: %s' % exclude)
+                excluded_models.add(model)
+            else:
+                try:
+                    app_config = apps.get_app_config(exclude)
+                except LookupError as e:
+                    raise CommandError(str(e))
+                excluded_apps.add(app_config)
+
+        if len(app_labels) == 0:
+            if primary_keys:
+                raise CommandError("You can only use --pks option with one model")
+            app_list = OrderedDict((app_config, None)
+                for app_config in apps.get_app_configs()
+                if app_config.models_module is not None and app_config not in excluded_apps)
+        else:
+            if len(app_labels) > 1 and primary_keys:
+                raise CommandError("You can only use --pks option with one model")
+            app_list = OrderedDict()
+            for label in app_labels:
+                try:
+                    app_label, model_label = label.split('.')
+                    try:
+                        app_config = apps.get_app_config(app_label)
+                    except LookupError as e:
+                        raise CommandError(str(e))
+                    if app_config.models_module is None or app_config in excluded_apps:
+                        continue
+                    try:
+                        model = app_config.get_model(model_label)
+                    except LookupError:
+                        raise CommandError("Unknown model: %s.%s" % (app_label, model_label))
+
+                    app_list_value = app_list.setdefault(app_config, [])
+
+                    # We may have previously seen a "all-models" request for
+                    # this app (no model qualifier was given). In this case
+                    # there is no need adding specific models to the list.
+                    if app_list_value is not None:
+                        if model not in app_list_value:
+                            app_list_value.append(model)
+                except ValueError:
+                    if primary_keys:
+                        raise CommandError("You can only use --pks option with one model")
+                    # This is just an app - no model qualifier
+                    app_label = label
+                    try:
+                        app_config = apps.get_app_config(app_label)
+                    except LookupError as e:
+                        raise CommandError(str(e))
+                    if app_config.models_module is None or app_config in excluded_apps:
+                        continue
+                    app_list[app_config] = None
+
+        # Check that the serialization format exists; this is a shortcut to
+        # avoid collating all the objects and _then_ failing.
+        if format not in serializers.get_public_serializer_formats():
+            try:
+                serializers.get_serializer(format)
+            except serializers.SerializerDoesNotExist:
+                pass
+
+            raise CommandError("Unknown serialization format: %s" % format)
+
+        def get_objects(count_only=False):
+            """
+            Collate the objects to be serialized. If count_only is True, just
+            count the number of objects to be serialized.
+            """
+            for model in serializers.sort_dependencies(app_list.items()):
+                if model in excluded_models:
+                    continue
+                if not model._meta.proxy and router.allow_migrate_model(using, model):
+                    if use_base_manager:
+                        objects = model._base_manager
+                    else:
+                        objects = model._default_manager
+
+                    # BEAT PATCH: let default order take place
+                    queryset = objects.using(using)
+                    # ORIGINAL LINE
+                    #queryset = objects.using(using).order_by(model._meta.pk.name)
+
+                    if primary_keys:
+                        queryset = queryset.filter(pk__in=primary_keys)
+                    if count_only:
+                        yield queryset.order_by().count()
+                    else:
+                        for obj in queryset.iterator():
+                            yield obj
+
+        try:
+            self.stdout.ending = None
+            progress_output = None
+            object_count = 0
+            # If dumpdata is outputting to stdout, there is no way to display progress
+            if (output and self.stdout.isatty() and options['verbosity'] > 0):
+                progress_output = self.stdout
+                object_count = sum(get_objects(count_only=True))
+            stream = open(output, 'w') if output else None
+            try:
+                serializers.serialize(format, get_objects(), indent=indent,
+                        use_natural_foreign_keys=use_natural_foreign_keys,
+                        use_natural_primary_keys=use_natural_primary_keys,
+                        stream=stream or self.stdout, progress_output=progress_output,
+                        object_count=object_count)
+            finally:
+                if stream:
+                    stream.close()
+        except Exception as e:
+            if show_traceback:
+                raise
+            raise CommandError("Unable to serialize database: %s" % e)