Skip to content
Snippets Groups Projects
Commit 42fc413e authored by André Anjos's avatar André Anjos :speech_balloon:
Browse files

Merge branch 'issue_433' into 'master' 

[many] Simplify backup system by using object's natural order (c.f. #433)



See merge request !199
parents aae3b726 51be8af7
No related branches found
No related tags found
1 merge request!199[many] Simplify backup system by using object's natural order (c.f. #433)
Pipeline #
Hide whitespace changes
Inline Side-by-side
beat/web/common/models.py
+ 3
− 2
View file @ 42fc413e
......@@ -381,7 +381,9 @@ class Versionable(Shareable):
class Meta(Shareable.Meta):
abstract = True
ordering = ['name', '-version']
# setup ordering so that the dump order respects self dependencies
ordering = ['previous_version_id', 'fork_of_id', 'id']
#_____ Static Methods __________
......@@ -552,7 +554,6 @@ class Contribution(Versionable):
class Meta(Versionable.Meta):
abstract = True
ordering = ['author__username', 'name', 'version']
unique_together = ('author', 'name', 'version')
......
beat/web/experiments/models.py
+ 3
− 0
View file @ 42fc413e
......@@ -926,6 +926,9 @@ class Block(models.Model):
class Meta:
unique_together = ('experiment', 'name')
# setup ordering so that the dump order respects self dependencies
ordering = ['id']
def __str__(self):
return self.experiment.fullname() + ', ' + self.name + ' (%s)' % self.get_status_display()
......
beat/web/utils/management/commands/backup.py
+ 2
− 17
View file @ 42fc413e
......@@ -149,7 +149,7 @@ class Command(BaseCommand):
destfile = os.path.join(tmpdir, 'initial.json')
logger.info("Dumping initial (unspecified) data -> `%s'", destfile)
arguments['output'] = destfile #new in Django-1.8.x
call_command('xdumpdata', **arguments)
call_command('dumpdata', **arguments)
# and backs-up the apps respecting the imposed order
for app in use_apps:
......@@ -158,23 +158,8 @@ class Command(BaseCommand):
arguments = copy.deepcopy(dump_arguments)
logger.info("Dumping data for `%s' -> `%s'", app, destfile)
if app in only:
app, model = only[app].split('.')
model = apps.get_model(app, model)
order = ('creation_date',)
# This will check and correct objects with weird creation
# dates so that the dump order is consistent
while True:
queryset = model.objects.order_by(*order)
err = _check(app, queryset)
if not err: break
arguments['primary_keys'] = ','.join([str(k.id) for k in queryset])
arguments['output'] = destfile #new in Django-1.8.x
call_command('xdumpdata', only.get(app, app), **arguments)
call_command('dumpdata', only.get(app, app), **arguments)
# copy prefix data
path = os.path.join(settings.PREFIX, app)
......
beat/web/utils/management/commands/xdumpdata.py deleted 100644 → 0
+ 0
− 187
View file @ aae3b726
# This file the Django/GitHub equivalent of
# django/core/management/commands/dumpdata.py. It was copied here so that we
# can implement primary-key ordering properly for our backup application. If
# you find this file is outdated, please re-download the appropriate Django
# version of this file and patch it up again. The patch area, which is quite
# small, is clearly indicated below.
#
# Django (BSD License)
from collections import OrderedDict
from django.apps import apps
from django.core import serializers
from django.core.management.base import BaseCommand, CommandError
from django.db import DEFAULT_DB_ALIAS, router
# #########################################################################
# Because this file was imported from Django's master, we need this import:
if not hasattr(serializers, 'sort_dependencies'):
# Covers Django 1.7 compatibility
from django.core.management.commands.dumpdata import sort_dependencies
serializers.sort_dependencies = sort_dependencies
routers.allow_migrate_model = routers.allow_migrate
# #########################################################################
class Command(BaseCommand):
help = ("Output the contents of the database as a fixture of the given "
"format (using each model's default manager unless --all is "
"specified).")
def add_arguments(self, parser):
parser.add_argument('args', metavar='app_label[.ModelName]', nargs='*',
help='Restricts dumped data to the specified app_label or app_label.ModelName.')
parser.add_argument('--format', default='json', dest='format',
help='Specifies the output serialization format for fixtures.')
parser.add_argument('--indent', default=None, dest='indent', type=int,
help='Specifies the indent level to use when pretty-printing output.')
parser.add_argument('--database', action='store', dest='database',
default=DEFAULT_DB_ALIAS,
help='Nominates a specific database to dump fixtures from. '
'Defaults to the "default" database.')
parser.add_argument('-e', '--exclude', dest='exclude', action='append', default=[],
help='An app_label or app_label.ModelName to exclude '
'(use multiple --exclude to exclude multiple apps/models).')
parser.add_argument('--natural-foreign', action='store_true', dest='use_natural_foreign_keys', default=False,
help='Use natural foreign keys if they are available.')
parser.add_argument('--natural-primary', action='store_true', dest='use_natural_primary_keys', default=False,
help='Use natural primary keys if they are available.')
parser.add_argument('-a', '--all', action='store_true', dest='use_base_manager', default=False,
help="Use Django's base manager to dump all models stored in the database, "
"including those that would otherwise be filtered or modified by a custom manager.")
parser.add_argument('--pks', dest='primary_keys',
help="Only dump objects with given primary keys. "
"Accepts a comma separated list of keys. "
"This option will only work when you specify one model.")
parser.add_argument('-o', '--output', default=None, dest='output',
help='Specifies file to which the output is written.')
def handle(self, *app_labels, **options):
format = options.get('format')
indent = options.get('indent')
using = options.get('database')
excludes = options.get('exclude')
output = options.get('output')
show_traceback = options.get('traceback')
use_natural_foreign_keys = options.get('use_natural_foreign_keys')
use_natural_primary_keys = options.get('use_natural_primary_keys')
use_base_manager = options.get('use_base_manager')
pks = options.get('primary_keys')
if pks:
primary_keys = pks.split(',')
else:
primary_keys = []
excluded_apps = set()
excluded_models = set()
for exclude in excludes:
if '.' in exclude:
try:
model = apps.get_model(exclude)
except LookupError:
raise CommandError('Unknown model in excludes: %s' % exclude)
excluded_models.add(model)
else:
try:
app_config = apps.get_app_config(exclude)
except LookupError as e:
raise CommandError(str(e))
excluded_apps.add(app_config)
if len(app_labels) == 0:
if primary_keys:
raise CommandError("You can only use --pks option with one model")
app_list = OrderedDict((app_config, None)
for app_config in apps.get_app_configs()
if app_config.models_module is not None and app_config not in excluded_apps)
else:
if len(app_labels) > 1 and primary_keys:
raise CommandError("You can only use --pks option with one model")
app_list = OrderedDict()
for label in app_labels:
try:
app_label, model_label = label.split('.')
try:
app_config = apps.get_app_config(app_label)
except LookupError as e:
raise CommandError(str(e))
if app_config.models_module is None or app_config in excluded_apps:
continue
try:
model = app_config.get_model(model_label)
except LookupError:
raise CommandError("Unknown model: %s.%s" % (app_label, model_label))
app_list_value = app_list.setdefault(app_config, [])
# We may have previously seen a "all-models" request for
# this app (no model qualifier was given). In this case
# there is no need adding specific models to the list.
if app_list_value is not None:
if model not in app_list_value:
app_list_value.append(model)
except ValueError:
if primary_keys:
raise CommandError("You can only use --pks option with one model")
# This is just an app - no model qualifier
app_label = label
try:
app_config = apps.get_app_config(app_label)
except LookupError as e:
raise CommandError(str(e))
if app_config.models_module is None or app_config in excluded_apps:
continue
app_list[app_config] = None
# Check that the serialization format exists; this is a shortcut to
# avoid collating all the objects and _then_ failing.
if format not in serializers.get_public_serializer_formats():
try:
serializers.get_serializer(format)
except serializers.SerializerDoesNotExist:
pass
raise CommandError("Unknown serialization format: %s" % format)
def get_objects():
# Collate the objects to be serialized.
for model in serializers.sort_dependencies(app_list.items()):
if model in excluded_models:
continue
if not model._meta.proxy and router.allow_migrate_model(using, model):
if use_base_manager:
objects = model._base_manager
else:
objects = model._default_manager
queryset = objects.using(using).order_by(model._meta.pk.name)
if primary_keys:
# ###################################################
# patch for BEAT starts here: order objects
# ###################################################
for pk in [int(k) for k in primary_keys]:
yield queryset.get(pk=pk)
else:
# ###################################################
# patch for BEAT ends here
# ###################################################
for obj in queryset.iterator():
yield obj
try:
self.stdout.ending = None
stream = open(output, 'w') if output else None
try:
serializers.serialize(format, get_objects(), indent=indent,
use_natural_foreign_keys=use_natural_foreign_keys,
use_natural_primary_keys=use_natural_primary_keys,
stream=stream or self.stdout)
finally:
if stream:
stream.close()
except Exception as e:
if show_traceback:
raise
raise CommandError("Unable to serialize database: %s" % e)
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment