From 38a9e29d1c5936cd1a2f1d07ef479db095cde336 Mon Sep 17 00:00:00 2001
From: Andre Anjos <andre.dos.anjos@gmail.com>
Date: Wed, 29 Jun 2016 15:18:26 +0200
Subject: [PATCH] Fix natural-key usage throughout scheduler branch mods
(closes #433)
---
beat/web/backend/models.py | 24 +++++++++++---------
beat/web/backend/utils.py | 8 +++----
beat/web/databases/models.py | 11 +++++++++
beat/web/experiments/models.py | 20 ++++++++++------
beat/web/utils/management/commands/backup.py | 11 ++++++---
5 files changed, 49 insertions(+), 25 deletions(-)
diff --git a/beat/web/backend/models.py b/beat/web/backend/models.py
index 4b6557714..b015b4c99 100644
--- a/beat/web/backend/models.py
+++ b/beat/web/backend/models.py
@@ -63,12 +63,12 @@ from ..statistics.utils import updateStatistics
class EnvironmentManager(ShareableManager):
- def get_by_natural_key(self, key):
- name, version = key.rsplit(' ', 1)
- return self.get(
- name=name,
- version=version[1:-1],
- )
+ def get_by_natural_key(self, name, version):
+ return self.get(name=name, version=version)
+
+ def get_by_fullname(self, fullname):
+ name, version = fullname.rsplit(' ', 1)
+ return self.get_by_natural_key(name, version[1:-1])
class Environment(Shareable):
@@ -128,7 +128,7 @@ class Environment(Shareable):
def natural_key(self):
- return self.fullname()
+ return (self.name, self.version)
#_____ Utilities __________
@@ -229,7 +229,7 @@ class Worker(models.Model):
def natural_key(self):
- return self.name
+ return (self.name,)
def get_admin_change_url(self):
@@ -303,7 +303,7 @@ class Worker(models.Model):
wishlist = Environment.objects.filter(queues__in=queues, active=True)
wishlist = wishlist.order_by('id').distinct()
- required = [k.natural_key() for k in wishlist]
+ required = [k.fullname() for k in wishlist]
missing = [k for k in required if k not in environments]
unused = [k for k in environments if k not in required]
@@ -532,6 +532,8 @@ class Queue(models.Model):
related_name='queues',
)
+ objects = QueueManager()
+
#_____ Meta parameters __________
class Meta:
@@ -553,7 +555,7 @@ class Queue(models.Model):
def natural_key(self):
- return self.name
+ return (self.name,)
def get_admin_change_url(self):
@@ -620,7 +622,7 @@ class Queue(models.Model):
'time-limit': self.time_limit,
'cores-per-slot': self.cores_per_slot,
'max-slots-per-user': self.max_slots_per_user,
- 'environments': [k.natural_key() for k in self.environments.all()],
+ 'environments': [k.fullname() for k in self.environments.all()],
'slots': dict([(s.worker.name, dict(quantity=s.quantity,
priority=s.priority)) for s in self.slots.all()]),
'groups': [k.name for k in Group.objects.all() if 'can_access' in get_perms(k, self)]
diff --git a/beat/web/backend/utils.py b/beat/web/backend/utils.py
index 02c91a991..bda58d941 100644
--- a/beat/web/backend/utils.py
+++ b/beat/web/backend/utils.py
@@ -280,7 +280,7 @@ def setup_backend(d):
# Associates environments with queues
for envkey in attrs['environments']:
- env = Environment.objects.get_by_natural_key(envkey)
+ env = Environment.objects.get_by_fullname(envkey)
logger.info("Appending `%s' to `%s'...", env, queue)
queue.environments.add(env)
@@ -317,7 +317,7 @@ def setup_backend(d):
# 10.3 Associate and dissociate environments
queue.environments.clear()
for e in attrs['environments']:
- env = Environment.objects.get_by_natural_key(e)
+ env = Environment.objects.get_by_fullname(e)
logger.info("Appending `%s' to `%s'...", env, queue)
queue.environments.add(env)
@@ -369,7 +369,7 @@ def find_environments(paths=None):
Returns:
dict: A dictionary containing each environment available using as key the
- natural key for environments (i.e., ``name (version)``) and as values
+ fullname for environments (i.e., ``name (version)``) and as values
another dictionary with these keys:
* name: The environment name (str)
@@ -407,5 +407,5 @@ def pick_execute(split, environments):
"""Resolves the path to the ``execute`` program to use for the split"""
# Check we have a compatible environment to execute the user algorithm
- envinfo = environments.get(split.job.block.environment.natural_key())
+ envinfo = environments.get(split.job.block.environment.fullname())
return envinfo['execute'] if envinfo else None
diff --git a/beat/web/databases/models.py b/beat/web/databases/models.py
index 1351621e1..01bb490b1 100755
--- a/beat/web/databases/models.py
+++ b/beat/web/databases/models.py
@@ -422,6 +422,12 @@ class DatabaseSetTemplateOutput(models.Model):
#----------------------------------------------------------
+class DatabaseSetOutputManager(models.Manager):
+
+ def get_by_natural_key(self, hash):
+ return self.get(hash=hash)
+
+
class DatabaseSetOutput(models.Model):
template = models.ForeignKey(DatabaseSetTemplateOutput,
related_name='instances', on_delete=models.CASCADE)
@@ -429,6 +435,8 @@ class DatabaseSetOutput(models.Model):
on_delete=models.CASCADE)
hash = models.CharField(max_length=64, unique=True)
+ objects = DatabaseSetOutputManager()
+
def __str__(self):
return self.fullname()
@@ -445,3 +453,6 @@ class DatabaseSetOutput(models.Model):
def all_needed_dataformats(self):
return self.template.all_needed_dataformats()
+
+ def natural_key(self):
+ return (self.hash,)
diff --git a/beat/web/experiments/models.py b/beat/web/experiments/models.py
index 67af2255b..51b5a60c4 100644
--- a/beat/web/experiments/models.py
+++ b/beat/web/experiments/models.py
@@ -1253,7 +1253,7 @@ class CachedFile(models.Model):
def natural_key(self):
- return self.hash
+ return (self.hash,)
def path(self):
@@ -1310,12 +1310,16 @@ class CachedFile(models.Model):
class BlockInputManager(models.Manager):
- def get_by_natural_key(self, hash):
- candidate = self.filter(cache__hash=hash)
- if candidate:
- return candidate[0]
+ def get_by_natural_key(self, name, experiment_author,
+ toolchain_author, toolchain_name,
+ toolchain_version, experiment_name, cache_hash, database_hash):
+ block = Block.objects.get_by_natural_key(name, experiment_author,
+ toolchain_author, toolchain_name, toolchain_version,
+ experiment_name)
+ if cache_hash:
+ return self.get(cache__hash=cache_hash, block=block)
else:
- return self.get(database__hash=hash)
+ return self.get(database__hash=database_hash, block=block)
class BlockInput(models.Model):
@@ -1337,7 +1341,9 @@ class BlockInput(models.Model):
objects = BlockInputManager()
def natural_key(self):
- return self.has,
+ cache_hash = self.cache and self.cache.hash
+ database_hash = self.database and self.database.hash
+ return self.block.natural_key() + (cache_hash, database_hash)
#----------------------------------------------------------
diff --git a/beat/web/utils/management/commands/backup.py b/beat/web/utils/management/commands/backup.py
index 5df5374de..bc3aa30ce 100644
--- a/beat/web/utils/management/commands/backup.py
+++ b/beat/web/utils/management/commands/backup.py
@@ -40,7 +40,7 @@ import datetime
from django.core.management import call_command
from django.core.management.base import BaseCommand
from django.conf import settings
-from django.apps import apps
+from django.apps import apps, registry
from ....import __version__
@@ -133,19 +133,24 @@ class Command(BaseCommand):
],
)
+ # remove uninstalled apps
+ global APPS
+ installed_apps = registry.apps.all_models.keys()
+ use_apps = [app for app in APPS if app in installed_apps]
+
try:
tmpdir = tempfile.mkdtemp('.backup', 'beat.web-')
# backs-up everything else first
arguments = copy.deepcopy(dump_arguments)
- arguments['exclude'] += APPS
+ arguments['exclude'] += use_apps
destfile = os.path.join(tmpdir, 'initial.json')
logger.info("Dumping initial (unspecified) data -> `%s'", destfile)
arguments['output'] = destfile #new in Django-1.8.x
call_command('xdumpdata', **arguments)
# and backs-up the apps respecting the imposed order
- for app in APPS:
+ for app in use_apps:
destfile = os.path.join(tmpdir, '%s.json' % app)
arguments = copy.deepcopy(dump_arguments)
--
GitLab