diff --git a/beat/web/attestations/tests.py b/beat/web/attestations/tests.py
index 28e0e2cb0d9b104a4934219a4617fb0278a6629b..58ac0e9db719e5078518cff1b390d6df06d55baf 100644
--- a/beat/web/attestations/tests.py
+++ b/beat/web/attestations/tests.py
@@ -221,7 +221,7 @@ class AttestationsAPIBase(BaseTestCase):
environment = Environment(name='env1', version='1.0')
environment.save()
- queue = Queue(name='queue1', memory_limit=1024, time_limit=60, nb_cores_per_slot=1, max_slots_per_user=10)
+ queue = Queue(name='queue1', memory_limit=1024, time_limit=60, cores_per_slot=1, max_slots_per_user=10)
queue.save()
queue.environments.add(environment)
@@ -280,18 +280,6 @@ class AttestationsAPIBase(BaseTestCase):
experiment.end_date = datetime.now()
experiment.save()
- block = Block()
- block.experiment = experiment
- block.name = 'block1'
- block.algorithm = algorithm1
- block.save()
-
- block = Block()
- block.experiment = experiment
- block.name = 'analyzer1'
- block.algorithm = algorithm2
- block.save()
-
def tearDown(self):
if os.path.exists(settings.TOOLCHAINS_ROOT):
diff --git a/beat/web/backend/api.py b/beat/web/backend/api.py
index 3316674a7c0da0e5cc4f3d956727c3525414b393..077b39ce93f71cb4f15deb570592ffe4316fcabf 100644
--- a/beat/web/backend/api.py
+++ b/beat/web/backend/api.py
@@ -27,7 +27,6 @@
from django.conf import settings
from django.db import transaction
-from django.db.models import Q
from django.utils import six
from rest_framework.decorators import api_view, permission_classes
@@ -510,13 +509,6 @@ def block_finished(request):
return BadRequestResponse("ERROR: The block '%s' is already marked as 'failed' in the database" % data['block-name'])
- # Create or retrieve cached files -- attach to block
- all_cached_files = []
- for hash in data['outputs']:
- cache = CachedFile.objects.get(hash=hash)
- cache.blocks.add(block)
- all_cached_files.append(cache)
-
# Updates all sorts of statistics on these caches (typically only one)
statistics = None
if block_state != 'cancelled':
@@ -526,53 +518,53 @@ def block_finished(request):
if data['statistics']:
statistics = beat.core.stats.Statistics(data['statistics'])
+ update_info = dict()
if statistics is not None:
- for cache in all_cached_files:
- cache.cpu_time = statistics.cpu['user'] + statistics.cpu['system']
- cache.max_memory = statistics.memory['rss']
- cache.data_read_size = statistics.data['volume']['read']
- cache.data_read_nb_blocks = statistics.data['blocks']['read']
- cache.data_read_time = statistics.data['time']['read']
- cache.data_written_size = statistics.data['volume']['write']
- cache.data_written_nb_blocks = statistics.data['blocks']['write']
- cache.data_written_time = statistics.data['time']['write']
+ update_info.update(dict(
+ cpu_time = statistics.cpu['user'] + statistics.cpu['system'],
+ max_memory = statistics.memory['rss'],
+ data_read_size = statistics.data['volume']['read'],
+ data_read_nb_blocks = statistics.data['blocks']['read'],
+ data_read_time = statistics.data['time']['read'],
+ data_written_size = statistics.data['volume']['write'],
+ data_written_nb_blocks = statistics.data['blocks']['write'],
+ data_written_time = statistics.data['time']['write'],
+ ))
if data.has_key('execution_info') and \
(data['execution_info'] is not None):
execution_infos = data['execution_info']
- for cache in all_cached_files:
-
- if execution_infos.has_key('linear_execution_time') and \
- (execution_infos['linear_execution_time'] is not None):
- cache.linear_execution_time = \
- execution_infos['linear_execution_time']
+ if execution_infos.has_key('linear_execution_time') and \
+ (execution_infos['linear_execution_time'] is not None):
+ update_info['linear_execution_time'] = \
+ execution_infos['linear_execution_time']
- if execution_infos.has_key('speed_up_real') and \
- (execution_infos['speed_up_real'] is not None):
- cache.speed_up_real = execution_infos['speed_up_real']
+ if execution_infos.has_key('speed_up_real') and \
+ (execution_infos['speed_up_real'] is not None):
+ update_info['speed_up_real'] = execution_infos['speed_up_real']
- if execution_infos.has_key('speed_up_maximal') and \
- (execution_infos['speed_up_maximal'] is not None):
- cache.speed_up_maximal = execution_infos['speed_up_maximal']
+ if execution_infos.has_key('speed_up_maximal') and \
+ (execution_infos['speed_up_maximal'] is not None):
+ update_info['speed_up_maximal'] = \
+ execution_infos['speed_up_maximal']
- if execution_infos.has_key('queuing_time') and \
- (execution_infos['queuing_time'] is not None):
- cache.queuing_time = execution_infos['queuing_time']
+ if execution_infos.has_key('queuing_time') and \
+ (execution_infos['queuing_time'] is not None):
+ update_info['queuing_time'] = execution_infos['queuing_time']
# Logged messages
if data.has_key('stdout') and isinstance(data['stdout'], six.string_types):
- for cache in all_cached_files: cache.stdout = data['stdout']
+ update_info['stdout'] = data['stdout']
if data.has_key('stderr') and isinstance(data['stderr'], six.string_types):
- for cache in all_cached_files: cache.stderr = data['stderr']
+ update_info['stderr'] = data['stderr']
if data.has_key('error-message') and \
isinstance(data['error-message'], six.string_types):
- for cache in all_cached_files:
- cache.error_report = data['error-message']
+ update_info['error_message'] = data['error-message']
# Saves all cached files to the database
- for cache in all_cached_files: cache.save()
+ CachedFile.objects.filter(hash__in=data['outputs']).update(**update_info)
# Save the results in the database (if applicable)
if block.analyzer and (block_state == 'processed'):
@@ -607,7 +599,8 @@ def block_finished(request):
block.status = Block.CACHED
block.save()
- if (experiment.status in [Experiment.SCHEDULED, Experiment.RUNNING]) and \
+ if (experiment.status in [Experiment.SCHEDULED, Experiment.RUNNING]) \
+ and \
(experiment.blocks.filter(analyzer=True).exclude(status=Block.CACHED).count() == 0):
experiment.end_date = datetime.now()
experiment.status = Experiment.DONE
@@ -627,12 +620,7 @@ def block_finished(request):
experiment.save()
elif block_state == 'cancelled':
- block.status = Block.NOT_CACHED
- block.statistics = None
- block.linear_execution_time = None
- block.speed_up_real = None
- block.speed_up_maximal = None
- block.queuing_time = None
+ block.status=Block.NOT_CACHED
block.save()
if experiment.status not in [Experiment.CANCELING, Experiment.FAILED]:
@@ -641,7 +629,7 @@ def block_finished(request):
experiment.save()
if experiment.status == Experiment.CANCELING:
- if experiment.blocks.filter(Q(status=Block.PROCESSING)).count() == 0:
+ if experiment.blocks.filter(status=Block.PROCESSING).count() == 0:
experiment.end_date = datetime.now()
experiment.status = Experiment.FAILED
experiment.save()
diff --git a/beat/web/backend/tests.py b/beat/web/backend/tests.py
index be1c7f3d4b48fbb0a2c6e305e0010964353be7a5..77f4fc9e07a2924a06f857b005bf9c163e96529a 100644
--- a/beat/web/backend/tests.py
+++ b/beat/web/backend/tests.py
@@ -153,7 +153,7 @@ class BackendAPIBase(BaseTestCase):
environment = Environment(name='env1', version='1.0')
environment.save()
- queue = Queue(name='queue1', memory_limit=1024, time_limit=60, nb_cores_per_slot=1, max_slots_per_user=10)
+ queue = Queue(name='queue1', memory_limit=1024, time_limit=60, cores_per_slot=1, max_slots_per_user=10)
queue.save()
queue.environments.add(environment)
@@ -578,7 +578,7 @@ class BlockStartedAPI(BackendAPIBase):
self.checkResponse(response, 204)
- experiment = Experiment.objects.get(id=self.experiment.id)
+ experiment.refresh_from_db()
self.assertTrue(experiment.start_date is not None)
self.assertTrue(experiment.end_date is None)
@@ -675,12 +675,15 @@ class BlockFinishedAPI(BackendAPIBase):
def test_bad_notification_request_with_unknown_experiment(self):
self.client.login(username='scheduler', password='1234')
+ experiment = Experiment.objects.get(id=self.experiment.id)
+ block = experiment.blocks.get(name='addition1')
+
response = self.client.put(self.url,
json.dumps({
'experiment-name': 'johndoe/toolchain1/1/unknown',
'block-name': 'addition1',
'state': 'processed',
- 'outputs': ['deadbeef'],
+ 'outputs': [o.encode() for o in block.outputs.values_list('hash', flat=True)],
'statistics': None,
}), content_type='application/json')
@@ -778,18 +781,21 @@ class BlockFinishedAPI(BackendAPIBase):
def test__not_cached_block__processed(self):
self.client.login(username='scheduler', password='1234')
+ experiment = Experiment.objects.get(id=self.experiment.id)
+ block = experiment.blocks.get(name='addition1')
+
response = self.client.put(self.url,
json.dumps({
'experiment-name': 'johndoe/toolchain1/1/experiment1',
'block-name': 'addition1',
'state': 'processed',
- 'outputs': ['deadbeef'],
+ 'outputs': [o.encode() for o in block.outputs.values_list('hash', flat=True)],
'statistics': None,
}), content_type='application/json')
self.checkResponse(response, 204)
- experiment = Experiment.objects.get(id=self.experiment.id)
+ experiment.refresh_from_db()
self.assertTrue(experiment.start_date is not None)
self.assertTrue(experiment.end_date is None)
@@ -813,12 +819,15 @@ class BlockFinishedAPI(BackendAPIBase):
def test__not_cached_block__failed(self):
self.client.login(username='scheduler', password='1234')
+ experiment = Experiment.objects.get(id=self.experiment.id)
+ block = experiment.blocks.get(name='addition1')
+
response = self.client.put(self.url,
json.dumps({
'experiment-name': 'johndoe/toolchain1/1/experiment1',
'block-name': 'addition1',
'state': 'failed',
- 'outputs': ['deadbeef'],
+ 'outputs': [o.encode() for o in block.outputs.values_list('hash', flat=True)],
'statistics': None,
}), content_type='application/json')
@@ -828,12 +837,15 @@ class BlockFinishedAPI(BackendAPIBase):
def test__not_cached_block__cancelled(self):
self.client.login(username='scheduler', password='1234')
+ experiment = Experiment.objects.get(id=self.experiment.id)
+ block = experiment.blocks.get(name='addition1')
+
response = self.client.put(self.url,
json.dumps({
'experiment-name': 'johndoe/toolchain1/1/experiment1',
'block-name': 'addition1',
'state': 'cancelled',
- 'outputs': ['deadbeef'],
+ 'outputs': [o.encode() for o in block.outputs.values_list('hash', flat=True)],
'statistics': None,
}), content_type='application/json')
@@ -854,7 +866,7 @@ class BlockFinishedAPI(BackendAPIBase):
'experiment-name': 'johndoe/toolchain1/1/experiment1',
'block-name': 'addition1',
'state': 'processed',
- 'outputs': ['deadbeef'],
+ 'outputs': [o.encode() for o in block.outputs.values_list('hash', flat=True)],
'statistics': None,
}), content_type='application/json')
@@ -875,7 +887,7 @@ class BlockFinishedAPI(BackendAPIBase):
'experiment-name': 'johndoe/toolchain1/1/experiment1',
'block-name': 'addition1',
'state': 'failed',
- 'outputs': ['deadbeef'],
+ 'outputs': [o.encode() for o in block.outputs.values_list('hash', flat=True)],
'statistics': None,
}), content_type='application/json')
@@ -896,7 +908,7 @@ class BlockFinishedAPI(BackendAPIBase):
'experiment-name': 'johndoe/toolchain1/1/experiment1',
'block-name': 'addition1',
'state': 'cancelled',
- 'outputs': ['deadbeef'],
+ 'outputs': [o.encode() for o in block.outputs.values_list('hash', flat=True)],
'statistics': None,
}), content_type='application/json')
@@ -917,7 +929,7 @@ class BlockFinishedAPI(BackendAPIBase):
'experiment-name': 'johndoe/toolchain1/1/experiment1',
'block-name': 'addition1',
'state': 'processed',
- 'outputs': ['deadbeef'],
+ 'outputs': [o.encode() for o in block.outputs.values_list('hash', flat=True)],
'statistics': None,
}), content_type='application/json')
@@ -938,7 +950,7 @@ class BlockFinishedAPI(BackendAPIBase):
'experiment-name': 'johndoe/toolchain1/1/experiment1',
'block-name': 'addition1',
'state': 'failed',
- 'outputs': ['deadbeef'],
+ 'outputs': [o.encode() for o in block.outputs.values_list('hash', flat=True)],
'statistics': None,
}), content_type='application/json')
@@ -959,7 +971,7 @@ class BlockFinishedAPI(BackendAPIBase):
'experiment-name': 'johndoe/toolchain1/1/experiment1',
'block-name': 'addition1',
'state': 'cancelled',
- 'outputs': ['deadbeef'],
+ 'outputs': [o.encode() for o in block.outputs.values_list('hash', flat=True)],
'statistics': None,
}), content_type='application/json')
@@ -971,18 +983,21 @@ class BlockFinishedAPI(BackendAPIBase):
self.start_block('addition1')
+ experiment = Experiment.objects.get(id=self.experiment.id)
+ block = experiment.blocks.get(name='addition1')
+
response = self.client.put(self.url,
json.dumps({
'experiment-name': 'johndoe/toolchain1/1/experiment1',
'block-name': 'addition1',
'state': 'processed',
- 'outputs': ['deadbeef'],
+ 'outputs': [o.encode() for o in block.outputs.values_list('hash', flat=True)],
'statistics': None,
}), content_type='application/json')
self.checkResponse(response, 204)
- experiment = Experiment.objects.get(id=self.experiment.id)
+ experiment.refresh_from_db()
self.assertTrue(experiment.start_date is not None)
self.assertTrue(experiment.end_date is None)
@@ -1009,18 +1024,21 @@ class BlockFinishedAPI(BackendAPIBase):
self.start_block('addition1')
+ experiment = Experiment.objects.get(id=self.experiment.id)
+ block = experiment.blocks.get(name='addition1')
+
response = self.client.put(self.url,
json.dumps({
'experiment-name': 'johndoe/toolchain1/1/experiment1',
'block-name': 'addition1',
'state': 'failed',
- 'outputs': ['deadbeef'],
+ 'outputs': [o.encode() for o in block.outputs.values_list('hash', flat=True)],
'statistics': None,
}), content_type='application/json')
self.checkResponse(response, 204)
- experiment = Experiment.objects.get(id=self.experiment.id)
+ experiment.refresh_from_db()
self.assertTrue(experiment.start_date is not None)
self.assertTrue(experiment.end_date is not None)
@@ -1046,18 +1064,21 @@ class BlockFinishedAPI(BackendAPIBase):
self.start_block('addition1')
+ experiment = Experiment.objects.get(id=self.experiment.id)
+ block = experiment.blocks.get(name='addition1')
+
response = self.client.put(self.url,
json.dumps({
'experiment-name': 'johndoe/toolchain1/1/experiment1',
'block-name': 'addition1',
'state': 'cancelled',
- 'outputs': ['deadbeef'],
+ 'outputs': [o.encode() for o in block.outputs.values_list('hash', flat=True)],
'statistics': None,
}), content_type='application/json')
self.checkResponse(response, 204)
- experiment = Experiment.objects.get(id=self.experiment.id)
+ experiment.refresh_from_db()
self.assertTrue(experiment.creation_date is not None)
self.assertTrue(experiment.start_date is not None)
@@ -1073,18 +1094,21 @@ class BlockFinishedAPI(BackendAPIBase):
self.start_block('addition1')
self.start_block('addition2')
+ experiment = Experiment.objects.get(id=self.experiment.id)
+ block = experiment.blocks.get(name='addition1')
+
response = self.client.put(self.url,
json.dumps({
'experiment-name': 'johndoe/toolchain1/1/experiment1',
'block-name': 'addition1',
'state': 'cancelled',
- 'outputs': ['deadbeef'],
+ 'outputs': [o.encode() for o in block.outputs.values_list('hash', flat=True)],
'statistics': None,
}), content_type='application/json')
self.checkResponse(response, 204)
- experiment = Experiment.objects.get(id=self.experiment.id)
+ experiment.refresh_from_db()
self.assertTrue(experiment.start_date is not None)
self.assertTrue(experiment.end_date is not None)
diff --git a/beat/web/experiments/tests.py b/beat/web/experiments/tests.py
index de67b02555b79d4d1083787b683affcb6202c50f..8b08fe78915cde54b7bc92ae26332af3c619eb46 100644
--- a/beat/web/experiments/tests.py
+++ b/beat/web/experiments/tests.py
@@ -207,7 +207,7 @@ class ExperimentTestBase(BaseTestCase):
environment = Environment(name='env1', version='1.0')
environment.save()
- queue = Queue(name='queue1', memory_limit=1024, time_limit=60, nb_cores_per_slot=1, max_slots_per_user=10)
+ queue = Queue(name='queue1', memory_limit=1024, time_limit=60, cores_per_slot=1, max_slots_per_user=10)
queue.save()
queue.environments.add(environment)
@@ -548,7 +548,7 @@ class ExperimentCreationAPI(ExperimentTestBase):
self.assertTrue(experiment.start_date is None)
self.assertTrue(experiment.end_date is None)
self.assertEqual(experiment.status, Experiment.PENDING)
- self.assertEqual(experiment.blocks.count(), 0)
+ self.assertEqual(experiment.blocks.count(), 3)
#----------------------------------------------------------
@@ -816,24 +816,24 @@ class ExperimentStartingAPI(ExperimentTestBase):
self.assertFalse(block.analyzer)
self.assertEqual(0, block.results.count())
- hashes = block.hashes.all()
- self.assertEqual(0, hashes.count())
+ hashes = block.outputs.all()
+ self.assertEqual(1, hashes.count())
block = experiment.blocks.get(name='addition2')
self.assertEqual(Block.NOT_CACHED, block.status)
self.assertFalse(block.analyzer)
self.assertEqual(0, block.results.count())
- hashes = block.hashes.all()
- self.assertEqual(0, hashes.count())
+ hashes = block.outputs.all()
+ self.assertEqual(1, hashes.count())
block = experiment.blocks.get(name='analysis')
self.assertEqual(Block.NOT_CACHED, block.status)
self.assertTrue(block.analyzer)
self.assertEqual(0, block.results.count())
- hashes = block.hashes.all()
- self.assertEqual(0, hashes.count())
+ hashes = block.outputs.all()
+ self.assertEqual(1, hashes.count())
def test_start_team_shared_experiment(self):
@@ -1102,11 +1102,9 @@ class ResultsAPI(ExperimentTestBase):
self.experiment.status = Experiment.FAILED
self.experiment.save()
- cached_file = CachedFile()
- cached_file.hash = 'deadbeef123456'
+ cached_file = block.first_cache()
cached_file.error_report = 'ERROR REPORT'
cached_file.save()
- cached_file.blocks.add(block)
self.client.login(username='johndoe', password='1234')
@@ -1190,7 +1188,7 @@ class ResultsAPI(ExperimentTestBase):
analysis_block.save()
db_result = Result()
- db_result.block = analysis_block
+ db_result.cache = analysis_block.first_cache()
db_result.name = 'out_float'
db_result.type = 'float32'
db_result.primary = True
@@ -1198,7 +1196,7 @@ class ResultsAPI(ExperimentTestBase):
db_result.save()
db_result = Result()
- db_result.block = analysis_block
+ db_result.cache = analysis_block.first_cache()
db_result.name = 'out_text'
db_result.type = 'string'
db_result.primary = True
@@ -1235,7 +1233,7 @@ class ResultsAPI(ExperimentTestBase):
analysis_block.save()
db_result = Result()
- db_result.block = analysis_block
+ db_result.cache = analysis_block.first_cache()
db_result.name = 'out_float'
db_result.type = 'float32'
db_result.primary = True
@@ -1243,7 +1241,7 @@ class ResultsAPI(ExperimentTestBase):
db_result.save()
db_result = Result()
- db_result.block = analysis_block
+ db_result.cache = analysis_block.first_cache()
db_result.name = 'out_text'
db_result.type = 'string'
db_result.primary = True
diff --git a/beat/web/reports/tests.py b/beat/web/reports/tests.py
index d9cbe6e6af909a4e3808811b03d529a551510df3..26839c9b5ade337706fa3f1786f5753e436dd654 100644
--- a/beat/web/reports/tests.py
+++ b/beat/web/reports/tests.py
@@ -511,7 +511,7 @@ class ReportTestCase(APITestCase):
environment = Environment(name='env1', version='1.0')
environment.save()
- queue = Queue(name='queue1', memory_limit=1024, time_limit=60, nb_cores_per_slot=1, max_slots_per_user=10)
+ queue = Queue(name='queue1', memory_limit=1024, time_limit=60, cores_per_slot=1, max_slots_per_user=10)
queue.save()
queue.environments.add(environment)