From 51be8af70a3a6ea6880f017e1514bb7df5c08c6a Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.anjos@idiap.ch>
Date: Thu, 30 Jun 2016 13:41:55 +0200
Subject: [PATCH] [many] Simplify backup system by using object's natural order
(c.f. #433)
---
beat/web/common/models.py | 5 +-
beat/web/experiments/models.py | 3 +
beat/web/utils/management/commands/backup.py | 19 +-
.../utils/management/commands/xdumpdata.py | 187 ------------------
4 files changed, 8 insertions(+), 206 deletions(-)
delete mode 100644 beat/web/utils/management/commands/xdumpdata.py
diff --git a/beat/web/common/models.py b/beat/web/common/models.py
index 66e680a07..9c5afc4b3 100644
--- a/beat/web/common/models.py
+++ b/beat/web/common/models.py
@@ -381,7 +381,9 @@ class Versionable(Shareable):
class Meta(Shareable.Meta):
abstract = True
- ordering = ['name', '-version']
+
+ # setup ordering so that the dump order respects self dependencies
+ ordering = ['previous_version_id', 'fork_of_id', 'id']
#_____ Static Methods __________
@@ -552,7 +554,6 @@ class Contribution(Versionable):
class Meta(Versionable.Meta):
abstract = True
- ordering = ['author__username', 'name', 'version']
unique_together = ('author', 'name', 'version')
diff --git a/beat/web/experiments/models.py b/beat/web/experiments/models.py
index 51b5a60c4..5e7b5a3d8 100644
--- a/beat/web/experiments/models.py
+++ b/beat/web/experiments/models.py
@@ -926,6 +926,9 @@ class Block(models.Model):
class Meta:
unique_together = ('experiment', 'name')
+ # setup ordering so that the dump order respects self dependencies
+ ordering = ['id']
+
def __str__(self):
return self.experiment.fullname() + ', ' + self.name + ' (%s)' % self.get_status_display()
diff --git a/beat/web/utils/management/commands/backup.py b/beat/web/utils/management/commands/backup.py
index 48706777b..c8791aeab 100644
--- a/beat/web/utils/management/commands/backup.py
+++ b/beat/web/utils/management/commands/backup.py
@@ -149,7 +149,7 @@ class Command(BaseCommand):
destfile = os.path.join(tmpdir, 'initial.json')
logger.info("Dumping initial (unspecified) data -> `%s'", destfile)
arguments['output'] = destfile #new in Django-1.8.x
- call_command('xdumpdata', **arguments)
+ call_command('dumpdata', **arguments)
# and backs-up the apps respecting the imposed order
for app in use_apps:
@@ -158,23 +158,8 @@ class Command(BaseCommand):
arguments = copy.deepcopy(dump_arguments)
logger.info("Dumping data for `%s' -> `%s'", app, destfile)
- if app in only:
-
- app, model = only[app].split('.')
- model = apps.get_model(app, model)
- order = ('creation_date',)
-
- # This will check and correct objects with weird creation
- # dates so that the dump order is consistent
- while True:
- queryset = model.objects.order_by(*order)
- err = _check(app, queryset)
- if not err: break
-
- arguments['primary_keys'] = ','.join([str(k.id) for k in queryset])
-
arguments['output'] = destfile #new in Django-1.8.x
- call_command('xdumpdata', only.get(app, app), **arguments)
+ call_command('dumpdata', only.get(app, app), **arguments)
# copy prefix data
path = os.path.join(settings.PREFIX, app)
diff --git a/beat/web/utils/management/commands/xdumpdata.py b/beat/web/utils/management/commands/xdumpdata.py
deleted file mode 100644
index e79e08049..000000000
--- a/beat/web/utils/management/commands/xdumpdata.py
+++ /dev/null
@@ -1,187 +0,0 @@
-# This file the Django/GitHub equivalent of
-# django/core/management/commands/dumpdata.py. It was copied here so that we
-# can implement primary-key ordering properly for our backup application. If
-# you find this file is outdated, please re-download the appropriate Django
-# version of this file and patch it up again. The patch area, which is quite
-# small, is clearly indicated below.
-#
-# Django (BSD License)
-
-from collections import OrderedDict
-
-from django.apps import apps
-from django.core import serializers
-from django.core.management.base import BaseCommand, CommandError
-from django.db import DEFAULT_DB_ALIAS, router
-
-# #########################################################################
-# Because this file was imported from Django's master, we need this import:
-if not hasattr(serializers, 'sort_dependencies'):
- # Covers Django 1.7 compatibility
- from django.core.management.commands.dumpdata import sort_dependencies
- serializers.sort_dependencies = sort_dependencies
- routers.allow_migrate_model = routers.allow_migrate
-# #########################################################################
-
-class Command(BaseCommand):
- help = ("Output the contents of the database as a fixture of the given "
- "format (using each model's default manager unless --all is "
- "specified).")
-
- def add_arguments(self, parser):
- parser.add_argument('args', metavar='app_label[.ModelName]', nargs='*',
- help='Restricts dumped data to the specified app_label or app_label.ModelName.')
- parser.add_argument('--format', default='json', dest='format',
- help='Specifies the output serialization format for fixtures.')
- parser.add_argument('--indent', default=None, dest='indent', type=int,
- help='Specifies the indent level to use when pretty-printing output.')
- parser.add_argument('--database', action='store', dest='database',
- default=DEFAULT_DB_ALIAS,
- help='Nominates a specific database to dump fixtures from. '
- 'Defaults to the "default" database.')
- parser.add_argument('-e', '--exclude', dest='exclude', action='append', default=[],
- help='An app_label or app_label.ModelName to exclude '
- '(use multiple --exclude to exclude multiple apps/models).')
- parser.add_argument('--natural-foreign', action='store_true', dest='use_natural_foreign_keys', default=False,
- help='Use natural foreign keys if they are available.')
- parser.add_argument('--natural-primary', action='store_true', dest='use_natural_primary_keys', default=False,
- help='Use natural primary keys if they are available.')
- parser.add_argument('-a', '--all', action='store_true', dest='use_base_manager', default=False,
- help="Use Django's base manager to dump all models stored in the database, "
- "including those that would otherwise be filtered or modified by a custom manager.")
- parser.add_argument('--pks', dest='primary_keys',
- help="Only dump objects with given primary keys. "
- "Accepts a comma separated list of keys. "
- "This option will only work when you specify one model.")
- parser.add_argument('-o', '--output', default=None, dest='output',
- help='Specifies file to which the output is written.')
-
- def handle(self, *app_labels, **options):
- format = options.get('format')
- indent = options.get('indent')
- using = options.get('database')
- excludes = options.get('exclude')
- output = options.get('output')
- show_traceback = options.get('traceback')
- use_natural_foreign_keys = options.get('use_natural_foreign_keys')
- use_natural_primary_keys = options.get('use_natural_primary_keys')
- use_base_manager = options.get('use_base_manager')
- pks = options.get('primary_keys')
-
- if pks:
- primary_keys = pks.split(',')
- else:
- primary_keys = []
-
- excluded_apps = set()
- excluded_models = set()
- for exclude in excludes:
- if '.' in exclude:
- try:
- model = apps.get_model(exclude)
- except LookupError:
- raise CommandError('Unknown model in excludes: %s' % exclude)
- excluded_models.add(model)
- else:
- try:
- app_config = apps.get_app_config(exclude)
- except LookupError as e:
- raise CommandError(str(e))
- excluded_apps.add(app_config)
-
- if len(app_labels) == 0:
- if primary_keys:
- raise CommandError("You can only use --pks option with one model")
- app_list = OrderedDict((app_config, None)
- for app_config in apps.get_app_configs()
- if app_config.models_module is not None and app_config not in excluded_apps)
- else:
- if len(app_labels) > 1 and primary_keys:
- raise CommandError("You can only use --pks option with one model")
- app_list = OrderedDict()
- for label in app_labels:
- try:
- app_label, model_label = label.split('.')
- try:
- app_config = apps.get_app_config(app_label)
- except LookupError as e:
- raise CommandError(str(e))
- if app_config.models_module is None or app_config in excluded_apps:
- continue
- try:
- model = app_config.get_model(model_label)
- except LookupError:
- raise CommandError("Unknown model: %s.%s" % (app_label, model_label))
-
- app_list_value = app_list.setdefault(app_config, [])
-
- # We may have previously seen a "all-models" request for
- # this app (no model qualifier was given). In this case
- # there is no need adding specific models to the list.
- if app_list_value is not None:
- if model not in app_list_value:
- app_list_value.append(model)
- except ValueError:
- if primary_keys:
- raise CommandError("You can only use --pks option with one model")
- # This is just an app - no model qualifier
- app_label = label
- try:
- app_config = apps.get_app_config(app_label)
- except LookupError as e:
- raise CommandError(str(e))
- if app_config.models_module is None or app_config in excluded_apps:
- continue
- app_list[app_config] = None
-
- # Check that the serialization format exists; this is a shortcut to
- # avoid collating all the objects and _then_ failing.
- if format not in serializers.get_public_serializer_formats():
- try:
- serializers.get_serializer(format)
- except serializers.SerializerDoesNotExist:
- pass
-
- raise CommandError("Unknown serialization format: %s" % format)
-
- def get_objects():
- # Collate the objects to be serialized.
- for model in serializers.sort_dependencies(app_list.items()):
- if model in excluded_models:
- continue
- if not model._meta.proxy and router.allow_migrate_model(using, model):
- if use_base_manager:
- objects = model._base_manager
- else:
- objects = model._default_manager
-
- queryset = objects.using(using).order_by(model._meta.pk.name)
- if primary_keys:
- # ###################################################
- # patch for BEAT starts here: order objects
- # ###################################################
- for pk in [int(k) for k in primary_keys]:
- yield queryset.get(pk=pk)
-
- else:
- # ###################################################
- # patch for BEAT ends here
- # ###################################################
- for obj in queryset.iterator():
- yield obj
-
- try:
- self.stdout.ending = None
- stream = open(output, 'w') if output else None
- try:
- serializers.serialize(format, get_objects(), indent=indent,
- use_natural_foreign_keys=use_natural_foreign_keys,
- use_natural_primary_keys=use_natural_primary_keys,
- stream=stream or self.stdout)
- finally:
- if stream:
- stream.close()
- except Exception as e:
- if show_traceback:
- raise
- raise CommandError("Unable to serialize database: %s" % e)
--
GitLab