Commit 756b797d authored by Philip ABBET's avatar Philip ABBET
Browse files

Merge updates & bugfixes

parents a718b102 bedb988d
......@@ -90,6 +90,13 @@ class AlgorithmModelForm(forms.ModelForm):
),
}
def __init__(self, *args, **kwargs):
super(AlgorithmModelForm, self).__init__(*args, **kwargs)
if kwargs['instance'].is_binary():
del self.fields['source_code_file']
def clean_declaration_file(self):
"""Cleans-up the file data, make sure it is really new"""
......@@ -231,4 +238,12 @@ class Algorithm(admin.ModelAdmin):
),
)
def get_form(self, request, obj=None, **kwargs):
if obj.is_binary():
self.exclude = ('source_code_file',)
fieldsets = filter(lambda x: x[0] == 'Definition',self.fieldsets)[0]
fieldsets[1]['fields'] = ('hash', 'splittable', 'declaration_file', 'language')
form = super(Algorithm, self).get_form(request, obj, **kwargs)
return form
admin.site.register(AlgorithmModel, Algorithm)
......@@ -288,7 +288,8 @@ function setupEditor(algorithm, dataformats, libraries)
};
{% if not binary %}
data.code = source_code_editor.getSourceCode();
if (declaration.language != 'cxx')
data.code = source_code_editor.getSourceCode();
{% endif %}
......
......@@ -42,7 +42,7 @@ import socket
CORES = psutil.cpu_count()
RAM = psutil.virtual_memory().total/(1024*1024)
ENVIRONMENT = {'name': 'environment', 'version': '1'}
ENVIRONMENT = {'name': 'Python 2.7', 'version': '1.1.0'}
CXX_ENVIRONMENT = {'name': 'Cxx backend', 'version': '1.0.0'}
ENVKEY = '%(name)s (%(version)s)' % ENVIRONMENT
CXX_ENVKEY = '%(name)s (%(version)s)' % CXX_ENVIRONMENT
......
......@@ -1365,7 +1365,7 @@ class JobSplit(models.Model):
self.end(result)
def process(self, cache=settings.CACHE_ROOT):
def process(self, cache=settings.CACHE_ROOT, docker_images_cache=None):
'''Process assigned job splits using beat.core
This task executes the user algorithm on a subprocess. It also serves
......@@ -1419,10 +1419,10 @@ class JobSplit(models.Model):
try:
if JobSplit.host is None:
JobSplit.host = Host()
JobSplit.host = Host(images_cache=docker_images_cache)
JobSplit.host.setup(raise_on_errors=not(getattr(settings, 'TEST_CONFIGURATION', False)))
self.executor = beat.core.execution.Executor(settings.PREFIX, config,
self.executor = beat.core.execution.Executor(JobSplit.host, settings.PREFIX, config,
cache)
if not self.executor.valid:
......@@ -1445,7 +1445,6 @@ class JobSplit(models.Model):
with self.executor:
self.start()
result = self.executor.process(
JobSplit.host,
virtual_memory_in_megabytes=queue.memory_limit,
max_cpu_percent=int(100*float(queue.cores_per_slot)), #allows for 150%
timeout_in_minutes=queue.time_limit,
......
......@@ -75,7 +75,7 @@ QUEUES_WITHOUT_PRIORITY = {
"time-limit": 180, #3 hours
"cores-per-slot": 1,
"max-slots-per-user": 4,
"environments": ['environment (1)'],
"environments": ['Python 2.7 (1.1.0)'],
"groups": [
"Default",
],
......@@ -92,7 +92,7 @@ QUEUES_WITHOUT_PRIORITY = {
"time-limit": 360, #6 hours
"cores-per-slot": 2,
"max-slots-per-user": 2,
"environments": ['environment (1)'],
"environments": ['Python 2.7 (1.1.0)'],
"groups": [
"Default",
],
......@@ -109,7 +109,7 @@ QUEUES_WITHOUT_PRIORITY = {
"time-limit": 720, #12 hours
"cores-per-slot": 4,
"max-slots-per-user": 1,
"environments": ['environment (1)'],
"environments": ['Python 2.7 (1.1.0)'],
"groups": [
"Default",
],
......@@ -129,9 +129,9 @@ QUEUES_WITHOUT_PRIORITY = {
}
},
"environments": {
'environment (1)': {
"name": 'environment',
"version": '1',
'Python 2.7 (1.1.0)': {
"name": 'Python 2.7',
"version": '1.1.0',
"short_description": "Test",
"description": "Test environment",
"languages": "python",
......@@ -147,7 +147,7 @@ PRIORITY_QUEUES = {
"time-limit": 180, #3 hours
"cores-per-slot": 1,
"max-slots-per-user": 2,
"environments": ['environment (1)'],
"environments": ['Python 2.7 (1.1.0)'],
"groups": [
"Default",
],
......@@ -168,7 +168,7 @@ PRIORITY_QUEUES = {
"time-limit": 360, #6 hours
"cores-per-slot": 2,
"max-slots-per-user": 1,
"environments": ['environment (1)'],
"environments": ['Python 2.7 (1.1.0)'],
"groups": [
"Default",
],
......@@ -189,7 +189,7 @@ PRIORITY_QUEUES = {
"time-limit": 180, #3 hours
"cores-per-slot": 1,
"max-slots-per-user": 8,
"environments": ['environment (1)'],
"environments": ['Python 2.7 (1.1.0)'],
"groups": [
"Default",
],
......@@ -219,9 +219,9 @@ PRIORITY_QUEUES = {
)
]),
"environments": {
'environment (1)': {
"name": 'environment',
"version": '1',
'Python 2.7 (1.1.0)': {
"name": 'Python 2.7',
"version": '1.1.0',
"short_description": "Test",
"description": "Test environment",
"languages": "python",
......@@ -383,7 +383,7 @@ class BaseBackendTestCase(TestCase):
setup_backend(qsetup.DEFAULT_CONFIGURATION)
Worker.objects.update(active=True)
env = Environment.objects.get(name='environment')
env = Environment.objects.get(name='Python 2.7')
queue = Queue.objects.first()
template_data = dict(
......@@ -420,7 +420,7 @@ class BaseBackendTestCase(TestCase):
self.assertEqual(b0.job.parent, None)
self.assertEqual(b0.job.child_, None)
self.assertEqual(b0.queue.name, 'queue')
self.assertEqual(b0.environment.name, 'environment')
self.assertEqual(b0.environment.name, 'Python 2.7')
self.assertEqual(b0.required_slots, 1)
self.assertEqual(b0.inputs.count(), 1)
self.assertEqual(b0.outputs.count(), 1)
......@@ -441,7 +441,7 @@ class BaseBackendTestCase(TestCase):
self.assertEqual(b1.job.parent, None)
self.assertEqual(b1.job.child_, None)
self.assertEqual(b1.queue.name, 'queue')
self.assertEqual(b1.environment.name, 'environment')
self.assertEqual(b1.environment.name, 'Python 2.7')
self.assertEqual(b1.required_slots, 1)
self.assertEqual(b1.inputs.count(), 1)
self.assertEqual(b1.outputs.count(), 1)
......@@ -527,8 +527,8 @@ class BackendSetup(BaseBackendTestCase):
env = q1.environments.first()
self.assertEqual(env.name, 'environment')
self.assertEqual(env.version, '1')
self.assertEqual(env.name, 'Python 2.7')
self.assertEqual(env.version, '1.1.0')
self.assertEqual(q1.slots.count(), 1)
self.assertEqual(q2.slots.count(), 1)
......@@ -636,8 +636,8 @@ class BackendSetup(BaseBackendTestCase):
env = q1.environments.first()
self.assertEqual(env.name, 'environment')
self.assertEqual(env.version, '1')
self.assertEqual(env.name, 'Python 2.7')
self.assertEqual(env.version, '1.1.0')
self.assertEqual(q1.slots.count(), 2)
self.assertEqual(q1_special.slots.count(), 2)
......@@ -1962,7 +1962,7 @@ class SchedulingPriority(BaseBackendTestCase):
q1 = Queue.objects.get(name='q1')
q2 = Queue.objects.get(name='q2')
env = Environment.objects.get(name='environment')
env = Environment.objects.get(name='Python 2.7')
# reset queue and environment to new backend configuration
self.set_globals(xp, q1, env)
......@@ -2009,7 +2009,7 @@ class SchedulingPriority(BaseBackendTestCase):
q1 = Queue.objects.get(name='q1')
q4 = Queue.objects.get(name='q4')
env = Environment.objects.get(name='environment')
env = Environment.objects.get(name='Python 2.7')
# reset queue and environment to new backend configuration
self.set_globals(xp, q1, env)
......@@ -2049,7 +2049,7 @@ class SchedulingPriority(BaseBackendTestCase):
Worker.objects.update(active=True)
q1 = Queue.objects.get(name='q1')
env = Environment.objects.get(name='environment')
env = Environment.objects.get(name='Python 2.7')
fullname = 'user/user/single/1/single'
xp = Experiment.objects.get(name=fullname.split(os.sep)[-1])
......@@ -2560,7 +2560,7 @@ class WorkingExternally(TransactionTestCase):
setup_backend(qsetup.DEFAULT_CONFIGURATION)
Worker.objects.update(active=True)
env = Environment.objects.get(name='environment')
env = Environment.objects.get(name='Python 2.7')
queue = Queue.objects.first()
template_data = dict(
......
......@@ -1169,16 +1169,23 @@ class Block(models.Model):
info = dict(
cpu_time = statistics.cpu['user'] + statistics.cpu['system'],
max_memory = statistics.memory['rss'],
data_read_size = statistics.data['volume']['read'],
data_read_nb_blocks = statistics.data['blocks']['read'],
data_read_time = statistics.data['time']['read'],
data_written_size = statistics.data['volume']['write'],
data_written_nb_blocks = statistics.data['blocks']['write'],
data_written_time = statistics.data['time']['write'],
stdout = result_stdout,
stderr = result_stderr,
error_report = self.job.result.usrerr,
)
)
if 'volume' in statistics.data:
info['data_read_size'] = statistics.data['volume'].get('read', 0)
info['data_written_size'] = statistics.data['volume'].get('write', 0)
if 'blocks' in statistics.data:
info['data_read_nb_blocks'] = statistics.data['blocks'].get('read', 0)
info['data_written_nb_blocks'] = statistics.data['blocks'].get('write', 0)
if 'time' in statistics.data:
info['data_read_time'] = statistics.data['time'].get('read', 0)
info['data_written_time'] = statistics.data['time'].get('write', 0)
if timings:
info.update(dict(
......
......@@ -1509,7 +1509,8 @@ beat.experiments.panels.Parameters.prototype.initialize = function(configuration
var iterator = this.environments.iterator();
var selected_environment_index = null;
var first_python_environment_index = null;
var python_environment_index = null;
var python_environment = null;
var default_environment = this.configuration.defaultEnvironment();
if (default_environment !== null)
......@@ -1533,8 +1534,17 @@ beat.experiments.panels.Parameters.prototype.initialize = function(configuration
selected_environment_index = environment_selector.children.length;
}
if ((first_python_environment_index === null) && (environment.languages.indexOf('python') >= 0))
first_python_environment_index = environment_selector.children.length;
if ((python_environment_index === null) && (environment.languages.indexOf('python') >= 0))
{
python_environment_index = environment_selector.children.length;
python_environment = environment;
}
else if ((python_environment_index !== null) && (python_environment.name == environment.name) &&
(python_environment.version < environment.version))
{
python_environment_index = environment_selector.children.length;
python_environment = environment;
}
environment_selector.appendChild(option);
selector_counter += 1;
......@@ -1542,8 +1552,8 @@ beat.experiments.panels.Parameters.prototype.initialize = function(configuration
if (selected_environment_index === null)
{
if (first_python_environment_index !== null)
selected_environment_index = first_python_environment_index;
if (python_environment_index !== null)
selected_environment_index = python_environment_index;
else
selected_environment_index = 0;
}
......
......@@ -115,4 +115,4 @@ def main(user_input=None):
signal.signal(signal.SIGTERM, handler)
signal.signal(signal.SIGINT, handler)
split.process()
split.process(docker_images_cache='/tmp/beat_docker_images.json')
......@@ -48,6 +48,7 @@ if 'beat.cmdline' in sys.argv:
LOGGING['handlers']['console']['level'] = 'DEBUG'
LOGGING['loggers']['beat.core']['handlers'] = ['discard']
LOGGING['loggers']['beat.web']['handlers'] = ['discard']
LOGGING['loggers']['beat.web.utils.management.commands']['handlers'] = ['discard']
PREFIX = os.environ.get('BEAT_TEST_PREFIX', os.path.realpath('./test_prefix'))
......
......@@ -58,11 +58,17 @@ def updateStatistics(stats, date=None):
# Modify the obj entry
obj.cpu_time += stats.cpu['user'] + stats.cpu['system']
obj.max_memory += stats.memory['rss']
obj.data_read_size += stats.data['volume']['read']
obj.data_read_nb_blocks += stats.data['blocks']['read']
obj.data_read_time += stats.data['time']['read']
obj.data_written_size += stats.data['volume']['write']
obj.data_written_nb_blocks += stats.data['blocks']['write']
obj.data_written_time += stats.data['time']['write']
if 'volume' in stats.data:
obj.data_read_size += stats.data['volume'].get('read', 0)
obj.data_written_size += stats.data['volume'].get('write', 0)
if 'blocks' in stats.data:
obj.data_read_nb_blocks += stats.data['blocks'].get('read', 0)
obj.data_written_nb_blocks += stats.data['blocks'].get('write', 0)
if 'time' in stats.data:
obj.data_read_time += stats.data['time'].get('read', 0)
obj.data_written_time += stats.data['time'].get('write', 0)
obj.save()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment