diff --git a/beat/web/algorithms/admin.py b/beat/web/algorithms/admin.py
old mode 100644
new mode 100755
index 50feac2e20a9a8a1bb5b2f09fe6ea791f3f921e4..f03412a9e5bece1e299670b6d1b0fd1a7039e81e
--- a/beat/web/algorithms/admin.py
+++ b/beat/web/algorithms/admin.py
@@ -90,6 +90,13 @@ class AlgorithmModelForm(forms.ModelForm):
),
}
+
+ def __init__(self, *args, **kwargs):
+ super(AlgorithmModelForm, self).__init__(*args, **kwargs)
+
+ if kwargs['instance'].is_binary():
+ del self.fields['source_code_file']
+
def clean_declaration_file(self):
"""Cleans-up the file data, make sure it is really new"""
@@ -231,4 +238,12 @@ class Algorithm(admin.ModelAdmin):
),
)
+ def get_form(self, request, obj=None, **kwargs):
+ if obj.is_binary():
+ self.exclude = ('source_code_file',)
+ fieldsets = filter(lambda x: x[0] == 'Definition',self.fieldsets)[0]
+ fieldsets[1]['fields'] = ('hash', 'splittable', 'declaration_file', 'language')
+ form = super(Algorithm, self).get_form(request, obj, **kwargs)
+ return form
+
admin.site.register(AlgorithmModel, Algorithm)
diff --git a/beat/web/algorithms/templates/algorithms/edition.html b/beat/web/algorithms/templates/algorithms/edition.html
index 91ebf64c877a613bcfb73ead50347cf1b1871144..4b6528cd186e778a31522ca19f4f3eb3ea89e12c 100644
--- a/beat/web/algorithms/templates/algorithms/edition.html
+++ b/beat/web/algorithms/templates/algorithms/edition.html
@@ -288,7 +288,8 @@ function setupEditor(algorithm, dataformats, libraries)
};
{% if not binary %}
- data.code = source_code_editor.getSourceCode();
+ if (declaration.language != 'cxx')
+ data.code = source_code_editor.getSourceCode();
{% endif %}
diff --git a/beat/web/backend/management/commands/qsetup.py b/beat/web/backend/management/commands/qsetup.py
index 837d93ce7ce3d618576b7fcfd0bf99f11f4255a8..48c4d3d474f7a31bf1303f0648ac8275490eb972 100755
--- a/beat/web/backend/management/commands/qsetup.py
+++ b/beat/web/backend/management/commands/qsetup.py
@@ -42,7 +42,7 @@ import socket
CORES = psutil.cpu_count()
RAM = psutil.virtual_memory().total/(1024*1024)
-ENVIRONMENT = {'name': 'environment', 'version': '1'}
+ENVIRONMENT = {'name': 'Python 2.7', 'version': '1.1.0'}
CXX_ENVIRONMENT = {'name': 'Cxx backend', 'version': '1.0.0'}
ENVKEY = '%(name)s (%(version)s)' % ENVIRONMENT
CXX_ENVKEY = '%(name)s (%(version)s)' % CXX_ENVIRONMENT
diff --git a/beat/web/backend/models.py b/beat/web/backend/models.py
index d7a62713d32b7890ead7dfb99107b06ef66ca7ef..1e2c3a74b9ccdf28504faca80db6cd8c56b2f8e7 100755
--- a/beat/web/backend/models.py
+++ b/beat/web/backend/models.py
@@ -1365,7 +1365,7 @@ class JobSplit(models.Model):
self.end(result)
- def process(self, cache=settings.CACHE_ROOT):
+ def process(self, cache=settings.CACHE_ROOT, docker_images_cache=None):
'''Process assigned job splits using beat.core
This task executes the user algorithm on a subprocess. It also serves
@@ -1419,10 +1419,10 @@ class JobSplit(models.Model):
try:
if JobSplit.host is None:
- JobSplit.host = Host()
+ JobSplit.host = Host(images_cache=docker_images_cache)
JobSplit.host.setup(raise_on_errors=not(getattr(settings, 'TEST_CONFIGURATION', False)))
- self.executor = beat.core.execution.Executor(settings.PREFIX, config,
+ self.executor = beat.core.execution.Executor(JobSplit.host, settings.PREFIX, config,
cache)
if not self.executor.valid:
@@ -1445,7 +1445,6 @@ class JobSplit(models.Model):
with self.executor:
self.start()
result = self.executor.process(
- JobSplit.host,
virtual_memory_in_megabytes=queue.memory_limit,
max_cpu_percent=int(100*float(queue.cores_per_slot)), #allows for 150%
timeout_in_minutes=queue.time_limit,
diff --git a/beat/web/backend/tests.py b/beat/web/backend/tests.py
index 263eb10294628aca81a40ab302787c5c750afe8d..aa4e82f74e83bab45e7df1c248d141a45d62cfda 100755
--- a/beat/web/backend/tests.py
+++ b/beat/web/backend/tests.py
@@ -75,7 +75,7 @@ QUEUES_WITHOUT_PRIORITY = {
"time-limit": 180, #3 hours
"cores-per-slot": 1,
"max-slots-per-user": 4,
- "environments": ['environment (1)'],
+ "environments": ['Python 2.7 (1.1.0)'],
"groups": [
"Default",
],
@@ -92,7 +92,7 @@ QUEUES_WITHOUT_PRIORITY = {
"time-limit": 360, #6 hours
"cores-per-slot": 2,
"max-slots-per-user": 2,
- "environments": ['environment (1)'],
+ "environments": ['Python 2.7 (1.1.0)'],
"groups": [
"Default",
],
@@ -109,7 +109,7 @@ QUEUES_WITHOUT_PRIORITY = {
"time-limit": 720, #12 hours
"cores-per-slot": 4,
"max-slots-per-user": 1,
- "environments": ['environment (1)'],
+ "environments": ['Python 2.7 (1.1.0)'],
"groups": [
"Default",
],
@@ -129,9 +129,9 @@ QUEUES_WITHOUT_PRIORITY = {
}
},
"environments": {
- 'environment (1)': {
- "name": 'environment',
- "version": '1',
+ 'Python 2.7 (1.1.0)': {
+ "name": 'Python 2.7',
+ "version": '1.1.0',
"short_description": "Test",
"description": "Test environment",
"languages": "python",
@@ -147,7 +147,7 @@ PRIORITY_QUEUES = {
"time-limit": 180, #3 hours
"cores-per-slot": 1,
"max-slots-per-user": 2,
- "environments": ['environment (1)'],
+ "environments": ['Python 2.7 (1.1.0)'],
"groups": [
"Default",
],
@@ -168,7 +168,7 @@ PRIORITY_QUEUES = {
"time-limit": 360, #6 hours
"cores-per-slot": 2,
"max-slots-per-user": 1,
- "environments": ['environment (1)'],
+ "environments": ['Python 2.7 (1.1.0)'],
"groups": [
"Default",
],
@@ -189,7 +189,7 @@ PRIORITY_QUEUES = {
"time-limit": 180, #3 hours
"cores-per-slot": 1,
"max-slots-per-user": 8,
- "environments": ['environment (1)'],
+ "environments": ['Python 2.7 (1.1.0)'],
"groups": [
"Default",
],
@@ -219,9 +219,9 @@ PRIORITY_QUEUES = {
)
]),
"environments": {
- 'environment (1)': {
- "name": 'environment',
- "version": '1',
+ 'Python 2.7 (1.1.0)': {
+ "name": 'Python 2.7',
+ "version": '1.1.0',
"short_description": "Test",
"description": "Test environment",
"languages": "python",
@@ -383,7 +383,7 @@ class BaseBackendTestCase(TestCase):
setup_backend(qsetup.DEFAULT_CONFIGURATION)
Worker.objects.update(active=True)
- env = Environment.objects.get(name='environment')
+ env = Environment.objects.get(name='Python 2.7')
queue = Queue.objects.first()
template_data = dict(
@@ -420,7 +420,7 @@ class BaseBackendTestCase(TestCase):
self.assertEqual(b0.job.parent, None)
self.assertEqual(b0.job.child_, None)
self.assertEqual(b0.queue.name, 'queue')
- self.assertEqual(b0.environment.name, 'environment')
+ self.assertEqual(b0.environment.name, 'Python 2.7')
self.assertEqual(b0.required_slots, 1)
self.assertEqual(b0.inputs.count(), 1)
self.assertEqual(b0.outputs.count(), 1)
@@ -441,7 +441,7 @@ class BaseBackendTestCase(TestCase):
self.assertEqual(b1.job.parent, None)
self.assertEqual(b1.job.child_, None)
self.assertEqual(b1.queue.name, 'queue')
- self.assertEqual(b1.environment.name, 'environment')
+ self.assertEqual(b1.environment.name, 'Python 2.7')
self.assertEqual(b1.required_slots, 1)
self.assertEqual(b1.inputs.count(), 1)
self.assertEqual(b1.outputs.count(), 1)
@@ -527,8 +527,8 @@ class BackendSetup(BaseBackendTestCase):
env = q1.environments.first()
- self.assertEqual(env.name, 'environment')
- self.assertEqual(env.version, '1')
+ self.assertEqual(env.name, 'Python 2.7')
+ self.assertEqual(env.version, '1.1.0')
self.assertEqual(q1.slots.count(), 1)
self.assertEqual(q2.slots.count(), 1)
@@ -636,8 +636,8 @@ class BackendSetup(BaseBackendTestCase):
env = q1.environments.first()
- self.assertEqual(env.name, 'environment')
- self.assertEqual(env.version, '1')
+ self.assertEqual(env.name, 'Python 2.7')
+ self.assertEqual(env.version, '1.1.0')
self.assertEqual(q1.slots.count(), 2)
self.assertEqual(q1_special.slots.count(), 2)
@@ -1962,7 +1962,7 @@ class SchedulingPriority(BaseBackendTestCase):
q1 = Queue.objects.get(name='q1')
q2 = Queue.objects.get(name='q2')
- env = Environment.objects.get(name='environment')
+ env = Environment.objects.get(name='Python 2.7')
# reset queue and environment to new backend configuration
self.set_globals(xp, q1, env)
@@ -2009,7 +2009,7 @@ class SchedulingPriority(BaseBackendTestCase):
q1 = Queue.objects.get(name='q1')
q4 = Queue.objects.get(name='q4')
- env = Environment.objects.get(name='environment')
+ env = Environment.objects.get(name='Python 2.7')
# reset queue and environment to new backend configuration
self.set_globals(xp, q1, env)
@@ -2049,7 +2049,7 @@ class SchedulingPriority(BaseBackendTestCase):
Worker.objects.update(active=True)
q1 = Queue.objects.get(name='q1')
- env = Environment.objects.get(name='environment')
+ env = Environment.objects.get(name='Python 2.7')
fullname = 'user/user/single/1/single'
xp = Experiment.objects.get(name=fullname.split(os.sep)[-1])
@@ -2560,7 +2560,7 @@ class WorkingExternally(TransactionTestCase):
setup_backend(qsetup.DEFAULT_CONFIGURATION)
Worker.objects.update(active=True)
- env = Environment.objects.get(name='environment')
+ env = Environment.objects.get(name='Python 2.7')
queue = Queue.objects.first()
template_data = dict(
diff --git a/beat/web/experiments/models.py b/beat/web/experiments/models.py
index 392382b9f6c2efb0acbe52e1c9dc26223fbe013d..9f9843b5ca02a69cdf6c22440e0f1409869aa59a 100755
--- a/beat/web/experiments/models.py
+++ b/beat/web/experiments/models.py
@@ -1169,16 +1169,23 @@ class Block(models.Model):
info = dict(
cpu_time = statistics.cpu['user'] + statistics.cpu['system'],
max_memory = statistics.memory['rss'],
- data_read_size = statistics.data['volume']['read'],
- data_read_nb_blocks = statistics.data['blocks']['read'],
- data_read_time = statistics.data['time']['read'],
- data_written_size = statistics.data['volume']['write'],
- data_written_nb_blocks = statistics.data['blocks']['write'],
- data_written_time = statistics.data['time']['write'],
stdout = result_stdout,
stderr = result_stderr,
error_report = self.job.result.usrerr,
- )
+ )
+
+ if 'volume' in statistics.data:
+ info['data_read_size'] = statistics.data['volume'].get('read', 0)
+ info['data_written_size'] = statistics.data['volume'].get('write', 0)
+
+ if 'blocks' in statistics.data:
+ info['data_read_nb_blocks'] = statistics.data['blocks'].get('read', 0)
+ info['data_written_nb_blocks'] = statistics.data['blocks'].get('write', 0)
+
+ if 'time' in statistics.data:
+ info['data_read_time'] = statistics.data['time'].get('read', 0)
+ info['data_written_time'] = statistics.data['time'].get('write', 0)
+
if timings:
info.update(dict(
diff --git a/beat/web/experiments/static/experiments/js/panels.js b/beat/web/experiments/static/experiments/js/panels.js
index aa0912f34cc5bc0bbfb98ef2941555bf828bf996..a52a0285ba9083c34cc77874858c29ff41c16a64 100644
--- a/beat/web/experiments/static/experiments/js/panels.js
+++ b/beat/web/experiments/static/experiments/js/panels.js
@@ -1509,7 +1509,8 @@ beat.experiments.panels.Parameters.prototype.initialize = function(configuration
var iterator = this.environments.iterator();
var selected_environment_index = null;
- var first_python_environment_index = null;
+ var python_environment_index = null;
+ var python_environment = null;
var default_environment = this.configuration.defaultEnvironment();
if (default_environment !== null)
@@ -1533,8 +1534,17 @@ beat.experiments.panels.Parameters.prototype.initialize = function(configuration
selected_environment_index = environment_selector.children.length;
}
- if ((first_python_environment_index === null) && (environment.languages.indexOf('python') >= 0))
- first_python_environment_index = environment_selector.children.length;
+ if ((python_environment_index === null) && (environment.languages.indexOf('python') >= 0))
+ {
+ python_environment_index = environment_selector.children.length;
+ python_environment = environment;
+ }
+ else if ((python_environment_index !== null) && (python_environment.name == environment.name) &&
+ (python_environment.version < environment.version))
+ {
+ python_environment_index = environment_selector.children.length;
+ python_environment = environment;
+ }
environment_selector.appendChild(option);
selector_counter += 1;
@@ -1542,8 +1552,8 @@ beat.experiments.panels.Parameters.prototype.initialize = function(configuration
if (selected_environment_index === null)
{
- if (first_python_environment_index !== null)
- selected_environment_index = first_python_environment_index;
+ if (python_environment_index !== null)
+ selected_environment_index = python_environment_index;
else
selected_environment_index = 0;
}
diff --git a/beat/web/scripts/process.py b/beat/web/scripts/process.py
old mode 100644
new mode 100755
index 8c44eba51343210169f9120c8d67b49a25f1fdf9..326a12b16267569ac8dd042087a66487c40ef60e
--- a/beat/web/scripts/process.py
+++ b/beat/web/scripts/process.py
@@ -115,4 +115,4 @@ def main(user_input=None):
signal.signal(signal.SIGTERM, handler)
signal.signal(signal.SIGINT, handler)
- split.process()
+ split.process(docker_images_cache='/tmp/beat_docker_images.json')
diff --git a/beat/web/settings/test.py b/beat/web/settings/test.py
old mode 100644
new mode 100755
index 4d45a26d9a48141bf8e2b1b6ae0d3f95ce8662a3..779aaa939479272aa3f501c5c5f3705e6f95f31d
--- a/beat/web/settings/test.py
+++ b/beat/web/settings/test.py
@@ -48,6 +48,7 @@ if 'beat.cmdline' in sys.argv:
LOGGING['handlers']['console']['level'] = 'DEBUG'
LOGGING['loggers']['beat.core']['handlers'] = ['discard']
+LOGGING['loggers']['beat.web']['handlers'] = ['discard']
LOGGING['loggers']['beat.web.utils.management.commands']['handlers'] = ['discard']
PREFIX = os.environ.get('BEAT_TEST_PREFIX', os.path.realpath('./test_prefix'))
diff --git a/beat/web/statistics/utils.py b/beat/web/statistics/utils.py
old mode 100644
new mode 100755
index bfca56829bccab0ec1ed66546842b6f5fb55f575..9a205a9bede1dcd2a9daddef0795c1ff7fefe81b
--- a/beat/web/statistics/utils.py
+++ b/beat/web/statistics/utils.py
@@ -58,11 +58,17 @@ def updateStatistics(stats, date=None):
# Modify the obj entry
obj.cpu_time += stats.cpu['user'] + stats.cpu['system']
obj.max_memory += stats.memory['rss']
- obj.data_read_size += stats.data['volume']['read']
- obj.data_read_nb_blocks += stats.data['blocks']['read']
- obj.data_read_time += stats.data['time']['read']
- obj.data_written_size += stats.data['volume']['write']
- obj.data_written_nb_blocks += stats.data['blocks']['write']
- obj.data_written_time += stats.data['time']['write']
+
+ if 'volume' in stats.data:
+ obj.data_read_size += stats.data['volume'].get('read', 0)
+ obj.data_written_size += stats.data['volume'].get('write', 0)
+
+ if 'blocks' in stats.data:
+ obj.data_read_nb_blocks += stats.data['blocks'].get('read', 0)
+ obj.data_written_nb_blocks += stats.data['blocks'].get('write', 0)
+
+ if 'time' in stats.data:
+ obj.data_read_time += stats.data['time'].get('read', 0)
+ obj.data_written_time += stats.data['time'].get('write', 0)
obj.save()