Fix natural-key usage throughout scheduler branch mods (closes #433)

beat/web/backend/models.py

@@ -63,12 +63,12 @@ from ..statistics.utils import updateStatistics
 
 class EnvironmentManager(ShareableManager):
 
-    def get_by_natural_key(self, key):
-        name, version = key.rsplit(' ', 1)
-        return self.get(
-            name=name,
-            version=version[1:-1],
-            )
+    def get_by_natural_key(self, name, version):
+        return self.get(name=name, version=version)
+
+    def get_by_fullname(self, fullname):
+        name, version = fullname.rsplit(' ', 1)
+        return self.get_by_natural_key(name, version[1:-1])
 
 
 class Environment(Shareable):
@@ -128,7 +128,7 @@ class Environment(Shareable):
 
 
     def natural_key(self):
-        return self.fullname()
+        return (self.name, self.version)
 
 
     #_____ Utilities __________
@@ -229,7 +229,7 @@ class Worker(models.Model):
 
 
     def natural_key(self):
-        return self.name
+        return (self.name,)
 
 
     def get_admin_change_url(self):
@@ -303,7 +303,7 @@ class Worker(models.Model):
         wishlist = Environment.objects.filter(queues__in=queues, active=True)
         wishlist = wishlist.order_by('id').distinct()
 
-        required = [k.natural_key() for k in wishlist]
+        required = [k.fullname() for k in wishlist]
         missing = [k for k in required if k not in environments]
         unused = [k for k in environments if k not in required]
 
@@ -532,6 +532,8 @@ class Queue(models.Model):
         related_name='queues',
         )
 
+    objects = QueueManager()
+
     #_____ Meta parameters __________
 
     class Meta:
@@ -553,7 +555,7 @@ class Queue(models.Model):
 
 
     def natural_key(self):
-        return self.name
+        return (self.name,)
 
 
     def get_admin_change_url(self):
@@ -620,7 +622,7 @@ class Queue(models.Model):
             'time-limit': self.time_limit,
             'cores-per-slot': self.cores_per_slot,
             'max-slots-per-user': self.max_slots_per_user,
-            'environments': [k.natural_key() for k in self.environments.all()],
+            'environments': [k.fullname() for k in self.environments.all()],
             'slots': dict([(s.worker.name, dict(quantity=s.quantity,
               priority=s.priority)) for s in self.slots.all()]),
             'groups': [k.name for k in Group.objects.all() if 'can_access' in get_perms(k, self)]

beat/web/backend/utils.py

@@ -280,7 +280,7 @@ def setup_backend(d):
 
         # Associates environments with queues
         for envkey in attrs['environments']:
-            env = Environment.objects.get_by_natural_key(envkey)
+            env = Environment.objects.get_by_fullname(envkey)
             logger.info("Appending `%s' to `%s'...", env, queue)
             queue.environments.add(env)
 
@@ -317,7 +317,7 @@ def setup_backend(d):
         # 10.3 Associate and dissociate environments
         queue.environments.clear()
         for e in attrs['environments']:
-            env = Environment.objects.get_by_natural_key(e)
+            env = Environment.objects.get_by_fullname(e)
             logger.info("Appending `%s' to `%s'...", env, queue)
             queue.environments.add(env)
 
@@ -369,7 +369,7 @@ def find_environments(paths=None):
   Returns:
 
     dict: A dictionary containing each environment available using as key the
-      natural key for environments (i.e., ``name (version)``) and as values
+      fullname for environments (i.e., ``name (version)``) and as values
       another dictionary with these keys:
 
         * name: The environment name (str)
@@ -407,5 +407,5 @@ def pick_execute(split, environments):
     """Resolves the path to the ``execute`` program to use for the split"""
 
     # Check we have a compatible environment to execute the user algorithm
-    envinfo = environments.get(split.job.block.environment.natural_key())
+    envinfo = environments.get(split.job.block.environment.fullname())
     return envinfo['execute'] if envinfo else None

beat/web/databases/models.py

@@ -422,6 +422,12 @@ class DatabaseSetTemplateOutput(models.Model):
 #----------------------------------------------------------
 
 
+class DatabaseSetOutputManager(models.Manager):
+
+    def get_by_natural_key(self, hash):
+        return self.get(hash=hash)
+
+
 class DatabaseSetOutput(models.Model):
     template = models.ForeignKey(DatabaseSetTemplateOutput,
         related_name='instances', on_delete=models.CASCADE)
@@ -429,6 +435,8 @@ class DatabaseSetOutput(models.Model):
         on_delete=models.CASCADE)
     hash = models.CharField(max_length=64, unique=True)
 
+    objects = DatabaseSetOutputManager()
+
     def __str__(self):
         return self.fullname()
 
@@ -445,3 +453,6 @@ class DatabaseSetOutput(models.Model):
 
     def all_needed_dataformats(self):
         return self.template.all_needed_dataformats()
+
+    def natural_key(self):
+        return (self.hash,)

beat/web/experiments/models.py

@@ -1253,7 +1253,7 @@ class CachedFile(models.Model):
 
 
     def natural_key(self):
-        return self.hash
+        return (self.hash,)
 
 
     def path(self):
@@ -1310,12 +1310,16 @@ class CachedFile(models.Model):
 
 class BlockInputManager(models.Manager):
 
-    def get_by_natural_key(self, hash):
-        candidate = self.filter(cache__hash=hash)
-        if candidate:
-            return candidate[0]
+    def get_by_natural_key(self, name, experiment_author,
+        toolchain_author, toolchain_name,
+        toolchain_version, experiment_name, cache_hash, database_hash):
+        block = Block.objects.get_by_natural_key(name, experiment_author,
+            toolchain_author, toolchain_name, toolchain_version,
+            experiment_name)
+        if cache_hash:
+            return self.get(cache__hash=cache_hash, block=block)
         else:
-            return self.get(database__hash=hash)
+            return self.get(database__hash=database_hash, block=block)
 
 
 class BlockInput(models.Model):
@@ -1337,7 +1341,9 @@ class BlockInput(models.Model):
     objects = BlockInputManager()
 
     def natural_key(self):
-        return self.has,
+        cache_hash = self.cache and self.cache.hash
+        database_hash = self.database and self.database.hash
+        return self.block.natural_key() + (cache_hash, database_hash)
 
 
 #----------------------------------------------------------

beat/web/utils/management/commands/backup.py

@@ -40,7 +40,7 @@ import datetime
 from django.core.management import call_command
 from django.core.management.base import BaseCommand
 from django.conf import settings
-from django.apps import apps
+from django.apps import apps, registry
 
 from ....import __version__
 
@@ -133,19 +133,24 @@ class Command(BaseCommand):
                     ],
                 )
 
+        # remove uninstalled apps
+        global APPS
+        installed_apps = registry.apps.all_models.keys()
+        use_apps = [app for app in APPS if app in installed_apps]
+
         try:
             tmpdir = tempfile.mkdtemp('.backup', 'beat.web-')
 
             # backs-up everything else first
             arguments = copy.deepcopy(dump_arguments)
-            arguments['exclude'] += APPS
+            arguments['exclude'] += use_apps
             destfile = os.path.join(tmpdir, 'initial.json')
             logger.info("Dumping initial (unspecified) data -> `%s'", destfile)
             arguments['output'] = destfile #new in Django-1.8.x
             call_command('xdumpdata', **arguments)
 
             # and backs-up the apps respecting the imposed order
-            for app in APPS:
+            for app in use_apps:
 
                 destfile = os.path.join(tmpdir, '%s.json' % app)
                 arguments = copy.deepcopy(dump_arguments)