From 005da01aedeb002bf19214bf355367e0d9db48e0 Mon Sep 17 00:00:00 2001
From: Philip ABBET <philip.abbet@idiap.ch>
Date: Wed, 23 Nov 2016 14:26:30 +0100
Subject: [PATCH] [db] Add supported languages to the 'backend.Environment'
model
---
.../migrations/0003_auto_20161123_1218.py | 20 +++++++++++
beat/web/backend/admin.py | 9 +++++
beat/web/backend/api.py | 2 ++
.../web/backend/management/commands/qsetup.py | 12 ++++++-
.../migrations/0004_environmentlanguage.py | 34 +++++++++++++++++++
beat/web/backend/models.py | 11 ++++++
beat/web/backend/tests.py | 10 +++---
beat/web/backend/utils.py | 17 ++++++++--
beat/web/code/models.py | 11 +++++-
beat/web/code/serializers.py | 2 +-
.../migrations/0003_auto_20161123_1218.py | 20 +++++++++++
.../migrations/0003_auto_20161123_1218.py | 20 +++++++++++
12 files changed, 159 insertions(+), 9 deletions(-)
create mode 100644 beat/web/algorithms/migrations/0003_auto_20161123_1218.py
mode change 100644 => 100755 beat/web/backend/admin.py
mode change 100644 => 100755 beat/web/backend/api.py
mode change 100644 => 100755 beat/web/backend/management/commands/qsetup.py
create mode 100644 beat/web/backend/migrations/0004_environmentlanguage.py
mode change 100644 => 100755 beat/web/backend/models.py
create mode 100644 beat/web/libraries/migrations/0003_auto_20161123_1218.py
create mode 100644 beat/web/plotters/migrations/0003_auto_20161123_1218.py
diff --git a/beat/web/algorithms/migrations/0003_auto_20161123_1218.py b/beat/web/algorithms/migrations/0003_auto_20161123_1218.py
new file mode 100644
index 000000000..da49951e7
--- /dev/null
+++ b/beat/web/algorithms/migrations/0003_auto_20161123_1218.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.11 on 2016-11-23 12:18
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('algorithms', '0002_cxx_backend'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='algorithm',
+ name='language',
+ field=models.CharField(choices=[(b'U', b'Unknown'), (b'C', b'Cxx'), (b'M', b'Matlab'), (b'P', b'Python'), (b'R', b'R')], default=b'P', max_length=1),
+ ),
+ ]
diff --git a/beat/web/backend/admin.py b/beat/web/backend/admin.py
old mode 100644
new mode 100755
index 8658a41a4..3a113b9e2
--- a/beat/web/backend/admin.py
+++ b/beat/web/backend/admin.py
@@ -29,6 +29,7 @@ from django.contrib import admin
from django import forms
from .models import Environment as EnvironmentModel
+from .models import EnvironmentLanguage as EnvironmentLanguageModel
from .models import Worker as WorkerModel
from .models import Queue as QueueModel
from .models import Slot as SlotModel
@@ -64,6 +65,10 @@ class EnvironmentModelForm(forms.ModelForm):
}
+class EnvironmentLanguageInline(admin.TabularInline):
+ model = EnvironmentLanguageModel
+
+
class Environment(admin.ModelAdmin):
list_display = (
@@ -87,6 +92,10 @@ class Environment(admin.ModelAdmin):
'name',
)
+ inlines = [
+ EnvironmentLanguageInline
+ ]
+
form = EnvironmentModelForm
filter_horizontal = [
diff --git a/beat/web/backend/api.py b/beat/web/backend/api.py
old mode 100644
new mode 100755
index 258472e8c..f0757c3b6
--- a/beat/web/backend/api.py
+++ b/beat/web/backend/api.py
@@ -30,6 +30,7 @@ from rest_framework.response import Response
from rest_framework import permissions
from .models import Environment
+from ..code.models import Code
@api_view(['GET'])
@@ -68,6 +69,7 @@ def accessible_environments_list(request):
'short_description': environment.short_description,
'queues': queues,
'accessibility': accessibility,
+ 'languages': [ Code.language_identifier(x.language) for x in environment.languages.iterator() ],
})
return Response(result)
diff --git a/beat/web/backend/management/commands/qsetup.py b/beat/web/backend/management/commands/qsetup.py
old mode 100644
new mode 100755
index 080d86964..f903d4524
--- a/beat/web/backend/management/commands/qsetup.py
+++ b/beat/web/backend/management/commands/qsetup.py
@@ -43,7 +43,9 @@ import socket
CORES = psutil.cpu_count()
RAM = psutil.virtual_memory().total/(1024*1024)
ENVIRONMENT = {'name': 'environment', 'version': '1'}
+CXX_ENVIRONMENT = {'name': 'cxx_environment', 'version': '1'}
ENVKEY = '%(name)s (%(version)s)' % ENVIRONMENT
+CXX_ENVKEY = '%(name)s (%(version)s)' % CXX_ENVIRONMENT
HOSTNAME = socket.gethostname()
DEFAULT_CONFIGURATION = {
@@ -53,7 +55,7 @@ DEFAULT_CONFIGURATION = {
"time-limit": 1440, #1 day
"cores-per-slot": 1,
"max-slots-per-user": CORES,
- "environments": [ENVKEY],
+ "environments": [CXX_ENVKEY, ENVKEY],
"slots": {
HOSTNAME: {
"quantity": CORES,
@@ -69,10 +71,18 @@ DEFAULT_CONFIGURATION = {
ENVKEY: {
"name": ENVIRONMENT['name'],
"version": ENVIRONMENT['version'],
+ "languages": ['python'],
"short_description": "Local python interpreter",
"description": "Automatically generated local python " \
"interpreter environment",
},
+ CXX_ENVKEY: {
+ "name": CXX_ENVIRONMENT['name'],
+ "version": CXX_ENVIRONMENT['version'],
+ "languages": ['cxx'],
+ "short_description": "C++ backend",
+ "description": "C++ backend running in a docker container",
+ },
},
"workers": {
HOSTNAME: {
diff --git a/beat/web/backend/migrations/0004_environmentlanguage.py b/beat/web/backend/migrations/0004_environmentlanguage.py
new file mode 100644
index 000000000..53a3594d3
--- /dev/null
+++ b/beat/web/backend/migrations/0004_environmentlanguage.py
@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.11 on 2016-11-23 12:18
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+def add_default_language(apps, schema_editor):
+ Environment = apps.get_model("backend", "Environment")
+ EnvironmentLanguage = apps.get_model("backend", "EnvironmentLanguage")
+
+ for env in Environment.objects.all():
+ lang = EnvironmentLanguage(language='P', environment=env)
+ lang.save()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('backend', '0003_remove_result_syserr'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='EnvironmentLanguage',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('language', models.CharField(choices=[(b'U', b'Unknown'), (b'C', b'Cxx'), (b'M', b'Matlab'), (b'P', b'Python'), (b'R', b'R')], default=b'P', max_length=1)),
+ ('environment', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='languages', to='backend.Environment')),
+ ],
+ ),
+ migrations.RunPython(add_default_language, migrations.RunPython.noop),
+ ]
diff --git a/beat/web/backend/models.py b/beat/web/backend/models.py
old mode 100644
new mode 100755
index d896979b1..eb6a447fb
--- a/beat/web/backend/models.py
+++ b/beat/web/backend/models.py
@@ -53,6 +53,7 @@ import beat.core.stats
import beat.core.data
import beat.core.execution
+from ..code.models import Code
from ..common.models import Shareable, ShareableManager
from ..common.texts import Messages
from ..statistics.utils import updateStatistics
@@ -164,6 +165,16 @@ class Environment(Shareable):
)
+class EnvironmentLanguage(models.Model):
+
+ environment = models.ForeignKey(Environment,
+ related_name='languages'
+ )
+
+ language = models.CharField(max_length=1, choices=Code.CODE_LANGUAGE,
+ default=Code.PYTHON)
+
+
#----------------------------------------------------------
diff --git a/beat/web/backend/tests.py b/beat/web/backend/tests.py
index cebfc4368..f51d93957 100755
--- a/beat/web/backend/tests.py
+++ b/beat/web/backend/tests.py
@@ -133,6 +133,7 @@ QUEUES_WITHOUT_PRIORITY = {
"version": '1',
"short_description": "Test",
"description": "Test environment",
+ "languages": "python",
},
},
}
@@ -222,6 +223,7 @@ PRIORITY_QUEUES = {
"version": '1',
"short_description": "Test",
"description": "Test environment",
+ "languages": "python",
},
},
}
@@ -380,7 +382,7 @@ class BaseBackendTestCase(TestCase):
setup_backend(qsetup.DEFAULT_CONFIGURATION)
Worker.objects.update(active=True)
- env = Environment.objects.first()
+ env = Environment.objects.get(name='environment')
queue = Queue.objects.first()
template_data = dict(
@@ -1959,7 +1961,7 @@ class SchedulingPriority(BaseBackendTestCase):
q1 = Queue.objects.get(name='q1')
q2 = Queue.objects.get(name='q2')
- env = Environment.objects.get()
+ env = Environment.objects.get(name='environment')
# reset queue and environment to new backend configuration
self.set_globals(xp, q1, env)
@@ -2006,7 +2008,7 @@ class SchedulingPriority(BaseBackendTestCase):
q1 = Queue.objects.get(name='q1')
q4 = Queue.objects.get(name='q4')
- env = Environment.objects.get()
+ env = Environment.objects.get(name='environment')
# reset queue and environment to new backend configuration
self.set_globals(xp, q1, env)
@@ -2046,7 +2048,7 @@ class SchedulingPriority(BaseBackendTestCase):
Worker.objects.update(active=True)
q1 = Queue.objects.get(name='q1')
- env = Environment.objects.get()
+ env = Environment.objects.get(name='environment')
fullname = 'user/user/single/1/single'
xp = Experiment.objects.get(name=fullname.split(os.sep)[-1])
diff --git a/beat/web/backend/utils.py b/beat/web/backend/utils.py
index 209cd9bdf..3ebfbe959 100755
--- a/beat/web/backend/utils.py
+++ b/beat/web/backend/utils.py
@@ -43,9 +43,10 @@ from django.db import transaction
from django.contrib.auth.models import Group
from guardian.shortcuts import assign_perm
+from ..code.models import Code
from ..common.models import Shareable
from ..experiments.models import CachedFile, Block, Experiment
-from .models import Queue, Worker, Job, Environment, Slot
+from .models import Queue, Worker, Job, Environment, EnvironmentLanguage, Slot
def cleanup_cache(path, age_in_minutes=0, delete=False):
@@ -219,6 +220,13 @@ def setup_backend(d):
logger.info("Creating `%s'...", env)
env.save()
+ for language in attrs['languages']:
+ lang = EnvironmentLanguage(
+ language=Code.language_db(language),
+ environment=env
+ )
+ lang.save()
+
# 8.1 Create new workers
config_workers = set(d['workers'].keys())
current_workers = set(Worker.objects.values_list('name', flat=True))
@@ -338,9 +346,14 @@ def setup_backend(d):
def dump_backend():
'''Returns a dictionary that represents the current backend configuration'''
+ environments = {}
+ for env in Environment.objects.all():
+ environments[str(env)] = env.as_dict()
+ environments[str(env)]['languages'] = [ Code.language_identifier(x.language) for x in env.languages.iterator() ]
+
return dict(
queues=dict([(k.name, k.as_dict()) for k in Queue.objects.all()]),
- environments=dict([(str(k), k.as_dict()) for k in Environment.objects.all()]),
+ environments=environments,
workers=dict([(k.name, k.as_dict()) for k in Worker.objects.all()]),
)
diff --git a/beat/web/code/models.py b/beat/web/code/models.py
index 33d98329b..c7053c6bf 100755
--- a/beat/web/code/models.py
+++ b/beat/web/code/models.py
@@ -487,7 +487,16 @@ class Code(StoredContribution):
def json_language(self):
- return filter(lambda x: x[0] == self.language, Code.CODE_LANGUAGE)[0][1].lower()
+ return Code.language_identifier(self.language)
+
+
+ @staticmethod
+ def language_identifier(db_language):
+ return filter(lambda x: x[0] == db_language, iter(Code.CODE_LANGUAGE))[0][1].lower()
+
+ @staticmethod
+ def language_db(language_identifier):
+ return filter(lambda x: x[1].lower() == language_identifier, iter(Code.CODE_LANGUAGE))[0][0]
#_____ Overrides __________
diff --git a/beat/web/code/serializers.py b/beat/web/code/serializers.py
index 9662168fe..08307391e 100755
--- a/beat/web/code/serializers.py
+++ b/beat/web/code/serializers.py
@@ -88,7 +88,7 @@ class CodeSerializer(ContributionSerializer):
return open_source
def get_language(self, obj):
- return filter(lambda x: x[0] == obj.language, iter(Code.CODE_LANGUAGE))[0][1].lower()
+ return Code.language_identifier(obj.language)
def get_accessibility(self, obj):
if obj.sharing == Code.PUBLIC:
diff --git a/beat/web/libraries/migrations/0003_auto_20161123_1218.py b/beat/web/libraries/migrations/0003_auto_20161123_1218.py
new file mode 100644
index 000000000..a9cf143a8
--- /dev/null
+++ b/beat/web/libraries/migrations/0003_auto_20161123_1218.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.11 on 2016-11-23 12:18
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('libraries', '0002_cxx_backend'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='library',
+ name='language',
+ field=models.CharField(choices=[(b'U', b'Unknown'), (b'C', b'Cxx'), (b'M', b'Matlab'), (b'P', b'Python'), (b'R', b'R')], default=b'P', max_length=1),
+ ),
+ ]
diff --git a/beat/web/plotters/migrations/0003_auto_20161123_1218.py b/beat/web/plotters/migrations/0003_auto_20161123_1218.py
new file mode 100644
index 000000000..7bae8a1cd
--- /dev/null
+++ b/beat/web/plotters/migrations/0003_auto_20161123_1218.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.11 on 2016-11-23 12:18
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('plotters', '0002_cxx_backend'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='plotter',
+ name='language',
+ field=models.CharField(choices=[(b'U', b'Unknown'), (b'C', b'Cxx'), (b'M', b'Matlab'), (b'P', b'Python'), (b'R', b'R')], default=b'P', max_length=1),
+ ),
+ ]
--
GitLab