Merge branch 'fix_backup_restore' into '1.4.x'

Fix backup restore

See merge request !236

.gitlab-ci.yml

@@ -3,6 +3,7 @@ stages:
 
 variables:
   PREFIX: /opt/beat.env.web/usr
+  TEST_PREFIX_PATH_FILE: '/tmp/test_prefix_$CI_JOB_ID.txt'
 
 build:
   stage: build
@@ -11,12 +12,13 @@ build:
   before_script:
   - ${PREFIX}/bin/python --version
   - docker info
+  - export BEAT_TEST_PREFIX=`mktemp -d --tmpdir=/var/tmp beat_test_prefix.XXXXXXXXX`
+  - echo $BEAT_TEST_PREFIX > $TEST_PREFIX_PATH_FILE
   script:
   - git clean -ffdx
   - ${PREFIX}/bin/python bootstrap-buildout.py
   - ./bin/buildout
   - export COVERAGE_FILE=.coverage.django
-  - export BEAT_TEST_PREFIX=`mktemp -d --tmpdir=/var/tmp beat_test_prefix.XXXXXXXXX`
   - ./bin/python ${PREFIX}/bin/coverage run --source=${CI_PROJECT_NAME} ./bin/django test --settings=beat.web.settings.ci -v 2
   - export BEAT_CMDLINE_TEST_PLATFORM=django://beat.web.settings.ci
   - export COVERAGE_FILE=.coverage.cmdline
@@ -24,13 +26,16 @@ build:
   - export NOSE_COVER_PACKAGE=beat.web
   - ./bin/python ${PREFIX}/bin/coverage run --source=./src/beat.cmdline ./bin/nosetests -sv beat.cmdline
   - unset COVERAGE_FILE
-  - rm -rf $BEAT_TEST_PREFIX
-  - unset BEAT_TEST_PREFIX
   - ./bin/python ${PREFIX}/bin/coverage combine .coverage.django .coverage.cmdline
   - ./bin/python ${PREFIX}/bin/coverage report
   - ./bin/python ${PREFIX}/bin/sphinx-apidoc --separate -d 2 --output=doc/api ${CI_PROJECT_NAMESPACE} beat/web/*/migrations beat/web/*/tests
   - ./bin/python ${PREFIX}/bin/sphinx-build doc/api html/api
   - ./bin/python ${PREFIX}/bin/sphinx-build doc/admin html/admin
   - ./bin/python ${PREFIX}/bin/sphinx-build doc/user html/user
+  after_script:
+  - export BEAT_TEST_PREFIX=`cat $TEST_PREFIX_PATH_FILE`
+  - echo "Deleting BEAT test prefix $BEAT_TEST_PREFIX"
+  - rm -rf $BEAT_TEST_PREFIX $TEST_PREFIX_PATH_FILE
+  - unset BEAT_TEST_PREFIX
   tags:
   - docker-build

beat/web/databases/models.py

@@ -301,6 +301,7 @@ class DatabaseProtocol(models.Model):
 
     def natural_key(self):
         return self.database.natural_key() + (self.name,)
+    natural_key.dependencies = ['databases.database']
 
     def fullname(self):
         if self.name != '':
@@ -394,6 +395,8 @@ class DatabaseSet(models.Model):
 
     def natural_key(self):
         return self.protocol.natural_key() + (self.name,) + self.template.natural_key()
+    natural_key.dependencies = ['databases.databaseprotocol',
+                                'databases.databasesettemplate']
 
     def fullname(self):
         if self.name != '':
@@ -437,15 +440,10 @@ class DatabaseSetTemplateOutput(models.Model):
 
 
 class DatabaseSetOutputManager(models.Manager):
-
-    def get_by_natural_key(self, database_name, database_version, protocol_name, name,
-                           template_name, output_name):
+    def get_by_natural_key(self, set_natural_key, output_name):
+        set_ = DatabaseSet.objects.get_by_natural_key(*set_natural_key)
         return self.get(
-            set__protocol__database__name=database_name,
-            set__protocol__database__version=database_version,
-            set__protocol__name=protocol_name,
-            set__name=name,
-            set__template__name=template_name,
+            set=set_,
             template__name=output_name,
         )
 
@@ -477,3 +475,5 @@ class DatabaseSetOutput(models.Model):
 
     def natural_key(self):
         return (self.set.natural_key(), self.template.name)
+    natural_key.dependencies = ['databases.databasesettemplateoutput',
+                                'databases.databaseset']

beat/web/experiments/models/block.py

@@ -138,6 +138,7 @@ class Block(models.Model):
             self.experiment.toolchain.version,
             self.experiment.name,
         )
+    natural_key.dependencies = ['experiments.experiment']
 
 
     def save(self, *args, **kwargs):

beat/web/experiments/models/block_input.py

@@ -26,32 +26,32 @@
 ###############################################################################
 
 from django.db import models
-from django.conf import settings
 
-from ...databases.models import DatabaseSetOutput
+from beat.web.databases.models import DatabaseSetOutput
 
 from .block import Block
 
 
-#----------------------------------------------------------
+# ----------------------------------------------------------
 
 
 class BlockInputManager(models.Manager):
 
-    def get_by_natural_key(self, name, experiment_author, toolchain_author,
-                           toolchain_name, toolchain_version, experiment_name,
-                           cache_hash, database_hash):
-        block = Block.objects.get_by_natural_key(name, experiment_author,
-                                                 toolchain_author, toolchain_name, toolchain_version,
-                                                 experiment_name)
+    def get_by_natural_key(self, block_natural_key,
+                           cache_hash, database_natural_key):
+        if block_natural_key:
+            block = Block.objects.get_by_natural_key(*block_natural_key)
+        else:
+            block = None
 
         if cache_hash:
             return self.get(cache__hash=cache_hash, block=block)
         else:
-            return self.get(database__hash=database_hash, block=block)
-
+            database = DatabaseSetOutput.objects.get_by_natural_key(
+                *database_natural_key)
+            return self.get(database=database, block=block)
 
-#----------------------------------------------------------
+# ----------------------------------------------------------
 
 
 class BlockInput(models.Model):
@@ -68,12 +68,19 @@ class BlockInput(models.Model):
                                  null=True, on_delete=models.CASCADE)
 
     channel = models.CharField(max_length=200, default='', blank=True,
-                               help_text="Synchronization channel within the toolchain")
+                               help_text="Synchronization channel within "
+                                         "the toolchain")
 
     objects = BlockInputManager()
 
-
     def natural_key(self):
+        if self.block:
+            block_natural_key = self.block.natural_key()
+        else:
+            block_natural_key = None
         cache_hash = self.cache and self.cache.hash
-        database_hash = self.database and self.database.hash
-        return self.block.natural_key() + (cache_hash, database_hash)
+        database_natural_key = self.database and self.database.natural_key()
+        return (block_natural_key, cache_hash, database_natural_key)
+    natural_key.dependencies = ['experiments.block',
+                                'experiments.cachedfile',
+                                'databases.databasesetoutput']

beat/web/experiments/models/experiment.py

@@ -278,9 +278,9 @@ class Experiment(Shareable):
     #_____ Utilities __________
 
     def natural_key(self):
-
         return (self.author.username,) + \
             self.toolchain.natural_key() + (self.name,)
+    natural_key.dependencies = ['toolchains.toolchain']
 
 
     #_____ Methods __________

beat/web/experiments/models/result.py

@@ -73,6 +73,7 @@ class Result(models.Model):
             self.name,
             self.cache.hash,
         )
+    natural_key.dependencies = ['experiments.cachedfile']
 
 
     def value(self):

beat/web/search/models.py

@@ -146,6 +146,7 @@ class Leaderboard(models.Model):
 
     def natural_key(self):
         return (self.search.author.username, self.search.name, self.search.version)
+    natural_key.dependencies = ['search.search']
 
     def get_absolute_url(self):
         return reverse(

buildout.cfg

@@ -88,7 +88,7 @@ eggs = alabaster
 
 [sources]
 beat.core = git git@gitlab.idiap.ch:beat/beat.core branch=1.6.x
-beat.cmdline = git git@gitlab.idiap.ch:beat/beat.cmdline
+beat.cmdline = git git@gitlab.idiap.ch:beat/beat.cmdline branch=1.4.x
 beat.backend.python = git git@gitlab.idiap.ch:beat/beat.backend.python branch=1.5.x
 beat.examples = git git@gitlab.idiap.ch:beat/beat.examples egg=false