diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index eb1c50659b3540f58eb2957b79d01fd8bf4cf444..b0d89ec11779bde51e988a2f797bdae74f4e9029 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,52 +1,36 @@
-py27-linux:
- image: docker.idiap.ch:5000/beat/beat.env.web:latest
- script:
- - /usr/local/beat/usr/bin/python bootstrap-buildout.py
- - ./bin/buildout
- - ./bin/python --version
- - export COVERAGE_FILE=.coverage.django
- - export BEAT_TEST_PREFIX=`mktemp -d --tmpdir=/var/tmp beat_test_prefix.XXXXXXXXX`
- - ./bin/coverage run --source='./beat/web' ./bin/django test --settings=beat.web.settings.test -v 2
- - export BEAT_CMDLINE_TEST_PLATFORM=django://beat.web.settings.test
- - export COVERAGE_FILE=.coverage.cmdline
- - export NOSE_WITH_COVERAGE=1
- - export NOSE_COVER_PACKAGE=beat.web
- - ./bin/nosetests -sv beat.cmdline
- - unset COVERAGE_FILE
- - rm -rf $BEAT_TEST_PREFIX
- - unset BEAT_TEST_PREFIX
- - ./bin/coverage combine .coverage.django .coverage.cmdline
- - ./bin/coverage report
- - ./bin/sphinx-apidoc --separate -d 2 --output=doc/api/api beat beat/web/*/migrations beat/web/*/tests
- - ./bin/sphinx-build doc/api html/api
- - ./bin/sphinx-build doc/admin html/admin
- - ./bin/sphinx-build doc/user html/user
+stages:
+ - build
- tags:
- - docker
+variables:
+ PREFIX: /opt/beat.env.web/usr
-py27-macosx:
+build:
+ stage: build
+ except:
+ - /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
+ before_script:
+ - ${PREFIX}/bin/python --version
+ - docker info
script:
- git clean -ffdx
- - /Users/buildbot/work/environments/beat/py27/bin/python bootstrap-buildout.py --setuptools-version=`/Users/buildbot/work/environments/beat/py27/bin/python -c 'import setuptools; print(setuptools.__version__)'`
+ - ${PREFIX}/bin/python bootstrap-buildout.py
- ./bin/buildout
- - ./bin/python --version
- - cd src/cpulimit && make && cd -
- - cd bin && ln -s ../src/cpulimit/src/cpulimit . && cd -
- export COVERAGE_FILE=.coverage.django
- - rm -rf ./test_prefix
- - ./bin/coverage run --source='./beat/web' ./bin/django test --settings=beat.web.settings.test -v 2
+ - export BEAT_TEST_PREFIX=`mktemp -d --tmpdir=/var/tmp beat_test_prefix.XXXXXXXXX`
+ - ./bin/python ${PREFIX}/bin/coverage run --source=${CI_PROJECT_NAME} ./bin/django test --settings=beat.web.settings.test -v 2
- export BEAT_CMDLINE_TEST_PLATFORM=django://beat.web.settings.test
- export COVERAGE_FILE=.coverage.cmdline
- export NOSE_WITH_COVERAGE=1
- export NOSE_COVER_PACKAGE=beat.web
- - ./bin/nosetests -sv beat.cmdline
+ - ./bin/python ${PREFIX}/bin/coverage run --source=./src/beat.cmdline ./bin/nosetests -sv beat.cmdline
- unset COVERAGE_FILE
- - ./bin/coverage combine .coverage.django .coverage.cmdline
- - ./bin/coverage report
- - ./bin/sphinx-apidoc --separate -d 2 --output=doc/api/api beat beat/web/*/migrations beat/web/*/tests
- - ./bin/sphinx-build doc/api html/api
- - ./bin/sphinx-build doc/admin html/admin
- - ./bin/sphinx-build doc/user html/user
+ - rm -rf $BEAT_TEST_PREFIX
+ - unset BEAT_TEST_PREFIX
+ - ./bin/python ${PREFIX}/bin/coverage combine .coverage.django .coverage.cmdline
+ - ./bin/python ${PREFIX}/bin/coverage report
+ - ./bin/python ${PREFIX}/bin/sphinx-apidoc --separate -d 2 --output=doc/api ${CI_PROJECT_NAMESPACE} beat/web/*/migrations beat/web/*/tests
+ - ./bin/python ${PREFIX}/bin/sphinx-build doc/api html/api
+ - ./bin/python ${PREFIX}/bin/sphinx-build doc/admin html/admin
+ - ./bin/python ${PREFIX}/bin/sphinx-build doc/user html/user
tags:
- - beat-macosx
+ - docker-build
diff --git a/beat/web/algorithms/api.py b/beat/web/algorithms/api.py
old mode 100644
new mode 100755
index e9b7c7723807f3a74800b827d5f45febaeee81b5..11a1e03fb55dde55fe8e20d991e0d70167521401
--- a/beat/web/algorithms/api.py
+++ b/beat/web/algorithms/api.py
@@ -25,6 +25,15 @@
# #
###############################################################################
+from django.http import Http404
+from django.http import HttpResponse
+from django.http import HttpResponseForbidden
+from django.http import HttpResponseBadRequest
+from django.shortcuts import get_object_or_404
+from django.conf import settings
+
+import os
+
from .models import Algorithm
from .serializers import AlgorithmSerializer
from .serializers import FullAlgorithmSerializer
@@ -118,3 +127,66 @@ class DiffAlgorithmView(DiffView):
"""
model = Algorithm
serializer_class = CodeDiffSerializer
+
+
+#----------------------------------------------------------
+
+
+def binary(request, author_name, object_name, version=None):
+ """Returns the shared library of a binary algorithm
+ """
+
+ if request.method not in ['GET', 'POST']:
+ return HttpResponseNotAllowed(['GET', 'POST'])
+
+ # Retrieves the algorithm
+ if version:
+ algorithm = get_object_or_404(
+ Algorithm,
+ author__username__iexact=author_name,
+ name__iexact=object_name,
+ version=int(version),
+ )
+ else:
+ algorithm = Algorithm.objects.filter(author__username__iexact=author_name,
+ name__iexact=object_name).order_by('-version')
+ if not algorithm:
+ raise Http404()
+ else:
+ algorithm = algorithm[0]
+
+ if not algorithm.is_binary():
+ raise Http404()
+
+ if request.method == 'GET':
+ (has_access, _, accessibility) = algorithm.accessibility_for(request.user, without_usable=True)
+
+ if not has_access:
+ raise Http404()
+
+ binary_data = algorithm.source_code
+
+ response = HttpResponse(binary_data, content_type='application/octet-stream')
+
+ response['Content-Length'] = len(binary_data)
+ response['Content-Disposition'] = 'attachment; filename=%d.so' % algorithm.version
+
+ return response
+
+ else:
+ if request.user.is_anonymous() or (request.user.username != author_name):
+ return HttpResponseForbidden()
+
+ if not request.FILES.has_key('binary'):
+ return HttpResponseBadRequest()
+
+ file = request.FILES['binary']
+
+ binary_data = ''
+ for chunk in file.chunks():
+ binary_data += chunk
+
+ algorithm.source_code = binary_data
+ algorithm.save()
+
+ return HttpResponse(status=204)
diff --git a/beat/web/algorithms/api_urls.py b/beat/web/algorithms/api_urls.py
old mode 100644
new mode 100755
index d12f252f7ae390c2d062bdb660f46b80565427e5..c0e84a2a1515693f0c6a34196e34ae521c7a56aa
--- a/beat/web/algorithms/api_urls.py
+++ b/beat/web/algorithms/api_urls.py
@@ -66,4 +66,14 @@ urlpatterns = [
name='object',
),
+ url(r'^(?P<author_name>\w+)/(?P<object_name>[-\w]+)/(?P<version>\d+)/binary/$',
+ api.binary,
+ name='binary',
+ ),
+
+ url(r'^(?P<author_name>\w+)/(?P<object_name>[-\w]+)/binary/$',
+ api.binary,
+ name='binary',
+ ),
+
]
diff --git a/beat/web/algorithms/migrations/0002_cxx_backend.py b/beat/web/algorithms/migrations/0002_cxx_backend.py
new file mode 100644
index 0000000000000000000000000000000000000000..abe1969adf9f9b37f82813da6237c71b66bfe482
--- /dev/null
+++ b/beat/web/algorithms/migrations/0002_cxx_backend.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.11 on 2016-11-09 11:05
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('algorithms', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='algorithm',
+ name='language',
+ field=models.CharField(choices=[(b'U', b'Unknown'), (b'C', b'C++'), (b'M', b'Matlab'), (b'P', b'Python'), (b'R', b'R')], default=b'P', max_length=1),
+ ),
+ ]
diff --git a/beat/web/algorithms/migrations/0003_auto_20161123_1218.py b/beat/web/algorithms/migrations/0003_auto_20161123_1218.py
new file mode 100644
index 0000000000000000000000000000000000000000..da49951e775d8f132c4758076f44d0a1513e2872
--- /dev/null
+++ b/beat/web/algorithms/migrations/0003_auto_20161123_1218.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.11 on 2016-11-23 12:18
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('algorithms', '0002_cxx_backend'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='algorithm',
+ name='language',
+ field=models.CharField(choices=[(b'U', b'Unknown'), (b'C', b'Cxx'), (b'M', b'Matlab'), (b'P', b'Python'), (b'R', b'R')], default=b'P', max_length=1),
+ ),
+ ]
diff --git a/beat/web/algorithms/models.py b/beat/web/algorithms/models.py
old mode 100644
new mode 100755
index 28864f02222654f25b90b1e62861c7af1f0b67a4..dbd68887bb63f06d3cd2552cb140b8ce2022a2cd
--- a/beat/web/algorithms/models.py
+++ b/beat/web/algorithms/models.py
@@ -307,6 +307,9 @@ class Algorithm(Code):
def deletable(self):
return (self.experiments.count() == 0) and super(Algorithm, self).deletable()
+ def valid(self):
+ return (self.source_code_file.name is not None) and (self.source_code_file.name != '')
+
def core(self):
return validate_algorithm(self.declaration)
diff --git a/beat/web/algorithms/static/algorithms/css/editor.css b/beat/web/algorithms/static/algorithms/css/editor.css
index 6a6c48aca0c61cc1111ab8061669e0ce9f43857d..a643f3dd62c8ca056edd3f964a94ac2714fd87bc 100644
--- a/beat/web/algorithms/static/algorithms/css/editor.css
+++ b/beat/web/algorithms/static/algorithms/css/editor.css
@@ -292,3 +292,16 @@ div.documentation_editor hr.separator
color: #C0C0C0;
background-color: #C0C0C0;
}
+
+
+div#binary_file_selector span.binary_file_button
+{
+ margin-left: 5px;
+ margin-right: 5px;
+}
+
+
+div#binary_file_selector span.filename
+{
+ font-style: italic;
+}
diff --git a/beat/web/algorithms/static/algorithms/js/models.js b/beat/web/algorithms/static/algorithms/js/models.js
index b231720468cfb225b5f83af8c77a6ea444c8e262..1b12b6416f797e692d5b64a6bd205c183239f045 100644
--- a/beat/web/algorithms/static/algorithms/js/models.js
+++ b/beat/web/algorithms/static/algorithms/js/models.js
@@ -52,6 +52,8 @@ beat.algorithms.models.Algorithm = function(data, duplicate_of)
this.outputs = data.outputs;
this.parameters = data.parameters;
this.splittable = data.splittable;
+ this.valid = data.valid;
+ this.language = data.language;
this.duplicate_of = (duplicate_of !== undefined ? duplicate_of : null);
}
@@ -152,8 +154,11 @@ beat.algorithms.models.Algorithm.prototype.outputsOfChannel = function(channel)
//
// declaration: JSON declaration of the list, as returned by the Web API
//----------------------------------------------------------------------------------------
-beat.algorithms.models.AlgorithmsList = function(declaration)
+beat.algorithms.models.AlgorithmsList = function(declaration, valid_only)
{
+ if (valid_only === undefined)
+ valid_only = false;
+
// Attributes
this.data = new Array();
@@ -168,6 +173,9 @@ beat.algorithms.models.AlgorithmsList = function(declaration)
{
var algorithm = declaration[i];
+ if (valid_only && !algorithm.valid)
+ continue;
+
if ((algorithm.name.indexOf('/private/') == -1) || (shared_hashes.indexOf(algorithm.hash) == -1))
{
this.data.push(new beat.algorithms.models.Algorithm(algorithm));
diff --git a/beat/web/algorithms/templates/algorithms/edition.html b/beat/web/algorithms/templates/algorithms/edition.html
index 21a91d30808bbe7e0f7ef53bd6cb795eabcef255..f97dfb546a03de4a79d5d6b3a6a81576eddd84be 100644
--- a/beat/web/algorithms/templates/algorithms/edition.html
+++ b/beat/web/algorithms/templates/algorithms/edition.html
@@ -36,6 +36,7 @@
{{ block.super }}
<link rel="stylesheet" href="{% fingerprint "algorithms/css/editor.css" %}" type="text/css" media="screen" />
<link rel="stylesheet" href="{% fingerprint "ui/css/smart-selector.css" %}" type="text/css" media="screen" />
+<link rel="stylesheet" href="{% fingerprint "blueimp-file-upload/css/jquery.fileupload.css" %}" type="text/css" media="screen" />
{% code_editor_css %}
{% endblock %}
@@ -46,6 +47,8 @@
<script src="{% fingerprint "algorithms/js/editor.js" %}" type="text/javascript" charset="utf-8"></script>
<script src="{% fingerprint "ui/js/smartselector.js" %}" type="text/javascript" charset="utf-8"></script>
+<script src="{% fingerprint "jquery-ui/ui/minified/jquery-ui.min.js" %}" type="text/javascript" charset="utf-8"></script>
+<script src="{% fingerprint "blueimp-file-upload/js/jquery.fileupload.js" %}" type="text/javascript" charset="utf-8"></script>
{% code_editor_scripts "python" %}
<script type="text/javascript">
@@ -73,9 +76,10 @@ function setupEditor(algorithm, dataformats, libraries)
{
$('.contribution_editor').show();
+{% if not binary %}
// Source code editor
var source_code_editor = new beat.contributions.editor.SourceCodeEditor('source_code');
-
+{% endif %}
// Options
var declaration = JSON.parse('{{ declaration|escapejs }}');
@@ -145,6 +149,7 @@ function setupEditor(algorithm, dataformats, libraries)
);
+{% if not binary %}
// Libraries editor
if (declaration.uses === undefined)
declaration.uses = {};
@@ -153,7 +158,7 @@ function setupEditor(algorithm, dataformats, libraries)
'libraries_editor', declaration.uses,
libraries, smart_selector
);
-
+{% endif %}
// Analyzer checkbox handling
checkbox_analyzer.change(function() {
@@ -172,11 +177,45 @@ function setupEditor(algorithm, dataformats, libraries)
results_panel.hide();
}
+{% if not binary %}
source_code_editor.changeProcessMethod(checkbox_analyzer[0].checked);
+{% endif %}
inputs_editor.setAnalyzer(checkbox_analyzer[0].checked);
});
+{% if not binary %}
+ // Language radio buttons handling
+ var language_python_selector = $('#language_python');
+ var language_cxx_selector = $('#language_cxx');
+
+ language_python_selector.change(function() {
+ $('#source_code_editor').show();
+ $('#libraries-editor-panel').show();
+ source_code_editor.editor.refresh();
+ });
+
+ language_cxx_selector.change(function() {
+ $('#source_code_editor').hide();
+ $('#libraries-editor-panel').hide();
+ });
+{% endif %}
+
+
+{% if binary %}
+ $('#shared_library').fileupload({
+ url: '{% url "api_algorithms:binary" algorithm_author algorithm_name algorithm_version %}',
+ add: function (e, data) {
+ $('.button_save')[0].shared_library = data;
+ $('#binary_file_name').text(data.files[0].name);
+ },
+ done: function (e, data) {
+ window.location = '{% url "algorithms:view" algorithm_author algorithm_name algorithm_version %}';
+ }
+ });
+{% endif %}
+
+
// Save button checkbox handling
$('.button_save').click(function() {
@@ -192,9 +231,16 @@ function setupEditor(algorithm, dataformats, libraries)
{% endif %}
{% endif %}
- var declaration = {
- language: 'python',
- };
+ var declaration = {};
+
+{% if not edition and not binary %}
+ if (language_python_selector[0].checked)
+ declaration.language = 'python';
+ else
+ declaration.language = 'cxx';
+{% else %}
+ declaration.language = '{{ algorithm_language }}';
+{% endif %}
if (!checkbox_analyzer[0].checked)
{
@@ -215,9 +261,36 @@ function setupEditor(algorithm, dataformats, libraries)
if (declaration.parameters === null)
return false;
- declaration.uses = libraries_panel.getLibraries(displayErrors);
- if (declaration.uses === null)
- return false;
+{% if not binary %}
+ if (language_python_selector[0].checked)
+ {
+ declaration.uses = libraries_panel.getLibraries(displayErrors);
+ if (declaration.uses === null)
+ return false;
+ }
+{% endif %}
+
+ var data = {
+ {% if not edition %}
+ {% if algorithm_version > 1 and not fork_of %}
+ name: '{{ algorithm_name }}',
+ previous_version:'{{ algorithm_author }}/{{ algorithm_name }}/{{ algorithm_version|add:-1 }}',
+ {% else %}
+ name: $('#algorithm_name')[0].value.trim(),
+ {% endif %}
+ description: (algorithm && algorithm.description) || '',
+ {% endif %}
+ short_description: (algorithm && algorithm.short_description) || '',
+ declaration: JSON.stringify(declaration),
+ {% if fork_of %}
+ fork_of: '{{ fork_of.fullname }}',
+ {% endif %}
+ };
+
+{% if not binary %}
+ if (language_python_selector[0].checked)
+ data.code = source_code_editor.getSourceCode();
+{% endif %}
$.ajax({
@@ -228,30 +301,22 @@ function setupEditor(algorithm, dataformats, libraries)
type: 'POST',
url: '{% url 'api_algorithms:list_create' algorithm_author %}',
{% endif %}
- data: JSON.stringify({
- {% if not edition %}
- {% if algorithm_version > 1 and not fork_of %}
- name: '{{ algorithm_name }}',
- previous_version:'{{ algorithm_author }}/{{ algorithm_name }}/{{ algorithm_version|add:-1 }}',
- {% else %}
- name: $('#algorithm_name')[0].value.trim(),
- {% endif %}
- description: (algorithm && algorithm.description) || '',
- {% endif %}
- short_description: (algorithm && algorithm.short_description) || '',
- declaration: JSON.stringify(declaration),
- code: source_code_editor.getSourceCode(),
- {% if fork_of %}
- fork_of: '{{ fork_of.fullname }}',
- {% endif %}
- }),
+ data: JSON.stringify(data),
contentType: "application/json; charset=utf-8",
dataType: "json",
success: function(data) {
- {% if edition %}
+ {% if not binary %}
+ {% if edition %}
window.location = '{% url "algorithms:view" algorithm_author algorithm_name algorithm_version %}';
+ {% else %}
+ window.location = data.object_view;
+ {% endif %}
{% else %}
+ {% if edition %}
+ $('.button_save')[0].shared_library.submit();
+ {% else %}
window.location = data.object_view;
+ {% endif %}
{% endif %}
},
error: function(jqXHR, textStatus, errorThrown) {
@@ -338,6 +403,17 @@ function setupEditor(algorithm, dataformats, libraries)
</div>
</div>
+{% if not edition and not binary %}
+<div class="row">
+ <div class="col-sm-offset-1 col-sm-10 contribution_editor" style="display: none;">
+ <label for="language_python">Language:</label>
+ <input id="language_python" type="radio" name="language" checked /><span class="label">Python</span>
+ <input id="language_cxx" type="radio" name="language" /><span class="label">C++</span>
+ <p class="help">The language you'll use to write this algorithm. Note that this cannot be changed later!</p>
+ </div>
+</div>
+{% endif %}
+
<div class="row">
<div class="col-sm-offset-1 col-sm-10">
@@ -449,6 +525,7 @@ function setupEditor(algorithm, dataformats, libraries)
{# libraries editor panel #}
+ {% if not binary %}
<div id="libraries-editor-panel" class="panel panel-default contribution_editor " style="display:none;">
<div class="panel-heading" role="tab" id="library-editor-1">
<h4 class="panel-title">
@@ -479,11 +556,13 @@ function setupEditor(algorithm, dataformats, libraries)
</div>{# panel-body #}
</div>{# panel-collapse #}
</div>{# panel #}
+ {% endif %}
</div>{# panel-group #}
</div>{# row #}
</div>
+{% if not binary %}
<div id="source_code_editor" class="row">
<div class="col-sm-offset-1 col-sm-10 contribution_editor" style="display: none;">
<label for="source_code">Source code:</label>
@@ -491,6 +570,35 @@ function setupEditor(algorithm, dataformats, libraries)
<p class="help">{{ messages.code|safe }}</p>
</div>
</div>
+{% elif edition %}
+<div id="binary_file_selector" class="row">
+ <div class="col-sm-offset-1 col-sm-10 contribution_editor" style="display: none;">
+ <div class="alert alert-info">
+ <i class="fa fa-info"></i> <span>This algorithm is implemented in <strong>{{ algorithm_language_name }}</strong>, compiled as a <strong>shared library</strong>.</span>
+ </div>
+ <label for="shared_library">Shared library:</label>
+ <span class="btn btn-success fileinput-button binary_file_button">
+ <i class="glyphicon glyphicon-plus"></i>
+ <span>Select file...</span>
+ <input id="shared_library" type="file" name="binary" />
+ </span>
+ <span id="binary_file_name" class="filename"></span>
+ <p class="help">{{ messages.shared_library|safe }}</p>
+ </div>
+</div>
+{% else %}
+<div class="row">
+ <div class="col-sm-offset-1 col-sm-10 contribution_editor" style="display: none;">
+ <div class="alert alert-info">
+ {% if new_version %}
+ <i class="fa fa-info"></i> <span>The algorithm is implemented in <strong>{{ algorithm_language_name }}</strong>, compiled as a <strong>shared library</strong>. The new algorithm version that will be created will copy the original shared library.</span>
+ {% else %}
+ <i class="fa fa-info"></i> <span>The original algorithm is implemented in <strong>{{ algorithm_language_name }}</strong>, compiled as a <strong>shared library</strong>. The forked algorithm that will be created will copy the original shared library.</span>
+ {% endif %}
+ </div>
+ </div>
+</div>
+{% endif %}
{% smart_selector "smart_selector" %}
diff --git a/beat/web/algorithms/templates/algorithms/panels/actions.html b/beat/web/algorithms/templates/algorithms/panels/actions.html
index 80d4bc6aaef571cb01bfb4d4dc342229661f233f..effba5f53684af32fbbbfb34cebd83f8a767e637 100644
--- a/beat/web/algorithms/templates/algorithms/panels/actions.html
+++ b/beat/web/algorithms/templates/algorithms/panels/actions.html
@@ -32,9 +32,11 @@
{% ifequal request.user.username object.author.username %}
<!-- Share, needs to be the owner and it may not be public already -->
+ {% if object.valid %}
{% ifnotequal object.get_sharing_display 'Public' %}
<a class="btn btn-default btn-share" href="{{ object.get_absolute_url }}#sharing" data-toggle="tooltip" data-placement="bottom" title="Share"><i class="fa fa-share-square-o fa-lg"></i></a>
{% endifnotequal %}
+ {% endif %}
<!-- Delete, needs to be the owner -->
{% if object.deletable %}
@@ -42,7 +44,9 @@
{% endif %}
<!-- New version, needs to be the owner -->
+ {% if object.valid %}
<a class="btn btn-default btn-new-version" href="{% url 'algorithms:new-version' object.name %}" data-toggle="tooltip" data-placement="bottom" title="New version"><i class="fa fa-copy fa-lg"></i></a>
+ {% endif %}
<!-- Edit, needs to be modifiable -->
{% if object.modifiable %}
@@ -56,12 +60,14 @@
<a class="btn btn-default btn-edit" href="{% url 'admin:algorithms_algorithm_change' object.id %}" data-toggle="tooltip" data-placement="bottom" title="Edit as admin"><i class="fa fa-cogs fa-lg"></i></a>
{% endif %}
- {% if open_source and not request.user.is_anonymous %}
+ {% if object.valid and open_source and not request.user.is_anonymous %}
<!-- Fork button, needs to be logged in -->
<a class="btn btn-default btn-fork" href="{% url 'algorithms:fork' object.author.username object.name object.version %}" data-toggle="tooltip" data-placement="bottom" title="Fork"><i class="fa fa-code-fork fa-lg"></i></a>
{% endif %}
<!-- Search, works for logged-in and anonymous users -->
+ {% if object.valid %}
<a class="btn btn-default btn-search" href="{% url 'search:search' %}?query=type:results%20{% if object.analysis %}analyzer{% else %}algo{% endif %}:{{ object.fullname }}" data-toggle="tooltip" data-placement="bottom" title="Search experiments"><i class="fa fa-search fa-lg"></i></a>
+ {% endif %}
</div>
diff --git a/beat/web/algorithms/templates/algorithms/panels/editor.html b/beat/web/algorithms/templates/algorithms/panels/editor.html
index 3a855db0da58a3f3efbc1dec8b5c9c992f8d14cc..c9603aaf7282d2af1bdb0e870bbbfc0aef3ebe39 100644
--- a/beat/web/algorithms/templates/algorithms/panels/editor.html
+++ b/beat/web/algorithms/templates/algorithms/panels/editor.html
@@ -228,9 +228,22 @@
{% endwith %}{# groups, uses or parameters #}
{% endwith %}{# groups, uses or parameters #}
- {% if open_source %}
+ {% if open_source and not object.is_binary %}
<textarea class="form-control" id="code-display">{{ object.source_code_file.read }}</textarea>
<p class="help-block">{{ texts.code|safe }}</p>
+ {% elif object.is_binary %}
+ {% if object.valid %}
+ <div class="alert alert-info">
+ <i class="fa fa-info"></i> <span>This algorithm is implemented in <strong>{{ object.language_fullname }}</strong>, compiled as a <strong>shared library</strong>.</span>
+ {% if downloadable %}
+ <a id="btn-download-binary" class="btn btn-success btn-sm" href="{% url "api_algorithms:binary" object.author.username object.name object.version %}"><i class="fa fa-download fa-lg"></i> Download</a>
+ {% endif %}
+ </div>
+ {% else %}
+ <div class="alert alert-warning">
+ <i class="fa fa-warning"></i> This algorithm must be implemented in <strong>{{ object.language_fullname }}</strong>, compiled as a <strong>shared library</strong> and uploaded to the platform.
+ </div>
+ {% endif %}
{% else %}
<div class="alert alert-warning">
<i class="fa fa-warning"></i> This algorithm is only usable to you. Its code was <strong>not</strong> shared.
@@ -246,7 +259,7 @@
</div>
-{% if open_source %}
+{% if open_source and not object.is_binary %}
<script type="text/javascript">
$(document).ready(function() {
var code_textarea = $('textarea#code-display');
diff --git a/beat/web/algorithms/templates/algorithms/panels/sharing.html b/beat/web/algorithms/templates/algorithms/panels/sharing.html
index 08afb2c953a8574610e875b25f221a08d148afca..467487ad346d11c61bc76960fe17c36513ebd3dd 100644
--- a/beat/web/algorithms/templates/algorithms/panels/sharing.html
+++ b/beat/web/algorithms/templates/algorithms/panels/sharing.html
@@ -128,6 +128,7 @@
algorithm</p>
</label>
</div>
+ {% if not object.is_binary %}
<div class="radio">
<label class="control-label">
<input id="shared-radio" type="radio" name="sharing" value="shared"/> Shared
@@ -143,6 +144,15 @@
code).</p>
</label>
</div>
+ {% else %}
+ <div class="radio">
+ <label class="control-label">
+ <input id="usable-radio" type="radio" name="sharing" value="usable"/> Usable
+ <p class="help">The users <b>and</b> teams indicated below will
+ be able to use this algorithm.</p>
+ </label>
+ </div>
+ {% endif %}
</div>
</fieldset>
<fieldset id="sharing-options" disabled>
diff --git a/beat/web/algorithms/templates/algorithms/view.html b/beat/web/algorithms/templates/algorithms/view.html
index cde3bb9696386d5baa9797315e2765656937c830..1998f2d4176760f93ebdcc141ecb992c80122f25 100644
--- a/beat/web/algorithms/templates/algorithms/view.html
+++ b/beat/web/algorithms/templates/algorithms/view.html
@@ -87,7 +87,7 @@
<ul id="object-tabs" class="nav nav-tabs" role="tablist">
<li role="presentation" class="active"><a href="#viewer" role="tab" data-toggle="tab" aria-controls="viewer">Algorithm</a></li>
<li role="presentation"><a {% if not algorithm.description %}title="No documentation available" {% endif %}href="#doc" role="tab" data-toggle="tab" aria-controls="doc">Documentation{% if not algorithm.description %} <i class="fa fa-warning"></i>{% endif %}</a></li>
- {% if owner %}
+ {% if owner and algorithm.valid %}
<li role="presentation"><a href="#sharing" role="tab" data-toggle="tab" aria-controls="sharing">Sharing</a></li>
{% endif %}
<li role="presentation"><a href="#experiments" role="tab" data-toggle="tab" aria-controls="experiments">Experiments <span class="badge">{{ experiments.count }}</span></a></li>
@@ -103,7 +103,7 @@
<div role="tabpanel" class="tab-pane" id="doc">
{% doc_editor algorithm 'api_algorithms:object' %}
</div>
- {% if owner %}
+ {% if owner and algorithm.valid %}
<div role="tabpanel" class="tab-pane" id="sharing">
{% algorithm_sharing algorithm %}
</div>
@@ -131,7 +131,7 @@
{% for key, value in execinfo %}<li class="list-group-item"><a title="Click to view" data-toggle="tooltip" data-placement="top" href="{% url 'backend:view-environment' key.name key.version %}">{{ key.fullname }}</a> <span class="badge">{{ value }}</span></li>{% endfor %}
</ul>
<p class="help">This table shows the number of times this algorithm
- has been <b>successfuly</b> run using the given environment. Note
+ has been <b>successfully</b> run using the given environment. Note
this does not provide sufficient information to evaluate if the
algorithm will run when submitted to different conditions.</p>
{% endif %}
diff --git a/beat/web/algorithms/templatetags/algorithm_tags.py b/beat/web/algorithms/templatetags/algorithm_tags.py
old mode 100644
new mode 100755
index 17e1bb0413c32aa6ff68852c959ba4e5f4a954ad..fb6caf386dd99bd03b65aabe2485bff6d00bdffe
--- a/beat/web/algorithms/templatetags/algorithm_tags.py
+++ b/beat/web/algorithms/templatetags/algorithm_tags.py
@@ -29,6 +29,7 @@
from django import template
from django.conf import settings
from ...common.texts import Messages as Texts
+from ...common.models import Shareable
register = template.Library()
@@ -89,6 +90,7 @@ def algorithm_editor(context, obj):
'object': obj,
'texts': Texts,
'open_source': obj.open_source(request.user),
+ 'downloadable': obj.is_binary() and ((request.user == obj.author) or (obj.sharing == Shareable.PUBLIC)),
}
diff --git a/beat/web/algorithms/tests/core.py b/beat/web/algorithms/tests/core.py
old mode 100644
new mode 100755
index 2f3c375bba9fa3262a42a298b7c97a599bf3382a..8bebec3fe64802df39d00b9040674991882533c3
--- a/beat/web/algorithms/tests/core.py
+++ b/beat/web/algorithms/tests/core.py
@@ -188,6 +188,7 @@ class AlgorithmsCreationFunction(AlgorithmsBaseTestCase):
class AlgorithmsTestCase(AlgorithmsBaseTestCase):
+
def setup_algorithms(self, declaration, code):
user1 = User.objects.get(username='jackdoe')
user2 = User.objects.get(username='johndoe')
@@ -304,6 +305,141 @@ class AlgorithmsTestCase(AlgorithmsBaseTestCase):
algorithm.share(public=True)
+ def setup_binary_algorithms(self, declaration, binary_data):
+ user1 = User.objects.get(username='jackdoe')
+ user2 = User.objects.get(username='johndoe')
+
+ team1 = Team.objects.get(name="teamdoe", owner=user1)
+ team2 = Team.objects.get(name="teamdoe2", owner=user1)
+
+ # Personal
+ (algorithm, errors) = Algorithm.objects.create_algorithm(
+ author=user1,
+ name='binary_personal',
+ short_description='',
+ declaration=declaration,
+ )
+ assert algorithm, errors
+ assert not algorithm.valid()
+
+ algorithm.source_code = binary_data
+ algorithm.save()
+
+ assert algorithm.valid()
+
+ # Usable by one user
+ (algorithm, errors) = Algorithm.objects.create_algorithm(
+ author=user1,
+ name='binary_usable_by_one_user',
+ short_description='',
+ declaration=declaration,
+ )
+ assert algorithm, errors
+
+ algorithm.source_code = binary_data
+ algorithm.save()
+
+ algorithm.share(public=False, users=[user2])
+
+ # Usable by one team
+ (algorithm, errors) = Algorithm.objects.create_algorithm(
+ author=user1,
+ name='binary_usable_by_one_team',
+ short_description='',
+ declaration=declaration,
+ )
+ assert algorithm, errors
+
+ algorithm.source_code = binary_data
+ algorithm.save()
+
+ algorithm.share(public=False, teams=[team1])
+
+ # Usable by teams
+ (algorithm, errors) = Algorithm.objects.create_algorithm(
+ author=user1,
+ name='binary_usable_by_teams',
+ short_description='',
+ declaration=declaration,
+ )
+ assert algorithm, errors
+
+ algorithm.source_code = binary_data
+ algorithm.save()
+
+ algorithm.share(public=False, teams=[team1, team2])
+
+ # Usable by all
+ (algorithm, errors) = Algorithm.objects.create_algorithm(
+ author=user1,
+ name='binary_usable_by_all',
+ short_description='',
+ declaration=declaration,
+ )
+ assert algorithm, errors
+
+ algorithm.source_code = binary_data
+ algorithm.save()
+
+ algorithm.share(public=False)
+
+ # Accessible to one user
+ (algorithm, errors) = Algorithm.objects.create_algorithm(
+ author=user1,
+ name='binary_public_for_one_user',
+ short_description='',
+ declaration=declaration,
+ )
+ assert algorithm, errors
+
+ algorithm.source_code = binary_data
+ algorithm.save()
+
+ algorithm.share(public=True, users=[user2])
+
+ # Accessible to one team
+ (algorithm, errors) = Algorithm.objects.create_algorithm(
+ author=user1,
+ name='binary_public_for_one_team',
+ short_description='',
+ declaration=declaration,
+ )
+ assert algorithm, errors
+
+ algorithm.source_code = binary_data
+ algorithm.save()
+
+ algorithm.share(public=True, teams=[team1])
+
+ # Accessible to teams
+ (algorithm, errors) = Algorithm.objects.create_algorithm(
+ author=user1,
+ name='binary_public_for_teams',
+ short_description='',
+ declaration=declaration,
+ )
+ assert algorithm, errors
+
+ algorithm.source_code = binary_data
+ algorithm.save()
+
+ algorithm.share(public=True, teams=[team1, team2])
+
+ # Accessible to all
+ (algorithm, errors) = Algorithm.objects.create_algorithm(
+ author=user1,
+ name='binary_public_for_all',
+ short_description='',
+ declaration=declaration,
+ )
+ assert algorithm, errors
+
+ algorithm.source_code = binary_data
+ algorithm.save()
+
+ algorithm.share(public=True)
+
+
class AlgorithmsAccessibilityFunctionsBase(AlgorithmsTestCase):
def setUp(self):
@@ -379,6 +515,27 @@ class AlgorithmsAPIBase(AlgorithmsTestCase):
JSON_DECLARATION = json.loads(DECLARATION)
+ CXX_DECLARATION = """{
+ "language": "cxx",
+ "splittable": false,
+ "groups": [
+ {
+ "name": "channel1",
+ "inputs": {
+ "a": { "type": "johndoe/single_integer/1" },
+ "b": { "type": "johndoe/single_integer/1" }
+ },
+ "outputs": {
+ "sum": { "type": "johndoe/single_integer/1" }
+ }
+ }
+ ],
+ "parameters": {
+ }
+}"""
+
+ JSON_CXX_DECLARATION = json.loads(CXX_DECLARATION)
+
CODE = """class Algorithm:
def process(self, inputs, outputs):
@@ -405,6 +562,13 @@ class AlgorithmsAPIBase(AlgorithmsTestCase):
}
}"""
+ BINARY = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F" \
+ "\xF0\xF1\xF2\xF3\xF4\xF5\xF6\xF7\xF8\xF9\xFA\xFB\xFC\xFD\xFE\xFF"
+
+ BINARY_UPDATE = "\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8A\x8B\x8C\x8D\x8E\x8F" \
+ "\xA0\xA1\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9\xAA\xAB\xAC\xAD\xAE\xAF" \
+ "\xC0\xC1\xC2\xC3\xC4\xC5\xC6\xC7\xC8\xC9\xCA\xCB\xCC\xCD\xCE\xCF"
+
def setUp(self):
super(AlgorithmsAPIBase, self).setUp()
@@ -446,3 +610,6 @@ class AlgorithmsAPIBase(AlgorithmsTestCase):
algorithm.previous_version = None
algorithm.fork_of = Algorithm.objects.get(name='public_for_all')
algorithm.save()
+
+ self.setup_binary_algorithms(AlgorithmsAPIBase.CXX_DECLARATION,
+ AlgorithmsAPIBase.BINARY)
diff --git a/beat/web/algorithms/tests/tests_api.py b/beat/web/algorithms/tests/tests_api.py
old mode 100644
new mode 100755
index 749fd77b85519cc9d8327615dd02e9e0b6341fde..1a25f26fa20eae20830ba18d87b6885c088f6e10
--- a/beat/web/algorithms/tests/tests_api.py
+++ b/beat/web/algorithms/tests/tests_api.py
@@ -31,9 +31,11 @@ import simplejson as json
from django.contrib.auth.models import User
from django.conf import settings
from django.core.urlresolvers import reverse
+from django.core.files.uploadedfile import SimpleUploadedFile
from ...dataformats.models import DataFormat
from ...common.testutils import tearDownModule
+from ...code.models import Code
import beat.core.algorithm
@@ -50,18 +52,34 @@ class AlgorithmsListRetrieval(AlgorithmsAPIBase):
content = self.checkResponse(response, 200, content_type='application/json')
self.assertTrue(isinstance(content, list))
- self.assertEqual(len(content), 2)
+ self.assertEqual(len(content), 4)
algorithm = filter(lambda x: x['name'] == 'jackdoe/public_for_all/1', content)[0]
self.assertEqual(algorithm['short_description'], '')
self.assertTrue(algorithm['fork_of'] is None)
self.assertEqual(algorithm['accessibility'], 'public')
+ self.assertEqual(algorithm['language'], 'python')
self.assertFalse(algorithm.has_key('is_owner'))
algorithm = filter(lambda x: x['name'] == 'jackdoe/usable_by_all/1', content)[0]
self.assertEqual(algorithm['short_description'], '')
self.assertTrue(algorithm['fork_of'] is None)
self.assertEqual(algorithm['accessibility'], 'confidential')
+ self.assertEqual(algorithm['language'], 'python')
+ self.assertFalse(algorithm.has_key('is_owner'))
+
+ algorithm = filter(lambda x: x['name'] == 'jackdoe/binary_public_for_all/1', content)[0]
+ self.assertEqual(algorithm['short_description'], '')
+ self.assertTrue(algorithm['fork_of'] is None)
+ self.assertEqual(algorithm['accessibility'], 'public')
+ self.assertEqual(algorithm['language'], 'cxx')
+ self.assertFalse(algorithm.has_key('is_owner'))
+
+ algorithm = filter(lambda x: x['name'] == 'jackdoe/binary_usable_by_all/1', content)[0]
+ self.assertEqual(algorithm['short_description'], '')
+ self.assertTrue(algorithm['fork_of'] is None)
+ self.assertEqual(algorithm['accessibility'], 'confidential')
+ self.assertEqual(algorithm['language'], 'cxx')
self.assertFalse(algorithm.has_key('is_owner'))
def test_all_accessible_algorithms(self):
@@ -73,7 +91,7 @@ class AlgorithmsListRetrieval(AlgorithmsAPIBase):
content = self.checkResponse(response, 200, content_type='application/json')
self.assertTrue(isinstance(content, list))
- self.assertEqual(len(content), 9)
+ self.assertEqual(len(content), 17)
algorithm = filter(lambda x: x['name'] == 'jackdoe/usable_by_one_user/1', content)[0]
self.assertEqual(algorithm['short_description'], '')
@@ -82,6 +100,7 @@ class AlgorithmsListRetrieval(AlgorithmsAPIBase):
self.assertTrue(algorithm['fork_of'] is None)
self.assertEqual(algorithm['is_owner'], False)
self.assertEqual(algorithm['accessibility'], 'confidential')
+ self.assertEqual(algorithm['language'], 'python')
algorithm = filter(lambda x: x['name'] == 'jackdoe/usable_by_one_team/1', content)[0]
self.assertEqual(algorithm['short_description'], '')
@@ -90,6 +109,7 @@ class AlgorithmsListRetrieval(AlgorithmsAPIBase):
self.assertTrue(algorithm['fork_of'] is None)
self.assertEqual(algorithm['is_owner'], False)
self.assertEqual(algorithm['accessibility'], 'confidential')
+ self.assertEqual(algorithm['language'], 'python')
algorithm = filter(lambda x: x['name'] == 'jackdoe/usable_by_teams/1', content)[0]
self.assertEqual(algorithm['short_description'], '')
@@ -98,6 +118,7 @@ class AlgorithmsListRetrieval(AlgorithmsAPIBase):
self.assertTrue(algorithm['fork_of'] is None)
self.assertEqual(algorithm['is_owner'], False)
self.assertEqual(algorithm['accessibility'], 'confidential')
+ self.assertEqual(algorithm['language'], 'python')
algorithm = filter(lambda x: x['name'] == 'jackdoe/public_for_all/1', content)[0]
self.assertEqual(algorithm['short_description'], '')
@@ -106,6 +127,7 @@ class AlgorithmsListRetrieval(AlgorithmsAPIBase):
self.assertTrue(algorithm['fork_of'] is None)
self.assertEqual(algorithm['is_owner'], False)
self.assertEqual(algorithm['accessibility'], 'public')
+ self.assertEqual(algorithm['language'], 'python')
algorithm = filter(lambda x: x['name'] == 'jackdoe/usable_by_all/1', content)[0]
self.assertEqual(algorithm['short_description'], '')
@@ -114,6 +136,7 @@ class AlgorithmsListRetrieval(AlgorithmsAPIBase):
self.assertTrue(algorithm['fork_of'] is None)
self.assertEqual(algorithm['is_owner'], False)
self.assertEqual(algorithm['accessibility'], 'confidential')
+ self.assertEqual(algorithm['language'], 'python')
algorithm = filter(lambda x: x['name'] == 'jackdoe/public_for_one_user/1', content)[0]
self.assertEqual(algorithm['short_description'], '')
@@ -122,6 +145,7 @@ class AlgorithmsListRetrieval(AlgorithmsAPIBase):
self.assertTrue(algorithm['fork_of'] is None)
self.assertEqual(algorithm['is_owner'], False)
self.assertEqual(algorithm['accessibility'], 'confidential')
+ self.assertEqual(algorithm['language'], 'python')
algorithm = filter(lambda x: x['name'] == 'jackdoe/public_for_one_team/1', content)[0]
self.assertEqual(algorithm['short_description'], '')
@@ -130,6 +154,7 @@ class AlgorithmsListRetrieval(AlgorithmsAPIBase):
self.assertTrue(algorithm['fork_of'] is None)
self.assertEqual(algorithm['is_owner'], False)
self.assertEqual(algorithm['accessibility'], 'confidential')
+ self.assertEqual(algorithm['language'], 'python')
algorithm = filter(lambda x: x['name'] == 'jackdoe/public_for_teams/1', content)[0]
self.assertEqual(algorithm['short_description'], '')
@@ -138,6 +163,7 @@ class AlgorithmsListRetrieval(AlgorithmsAPIBase):
self.assertTrue(algorithm['fork_of'] is None)
self.assertEqual(algorithm['is_owner'], False)
self.assertEqual(algorithm['accessibility'], 'confidential')
+ self.assertEqual(algorithm['language'], 'python')
algorithm = filter(lambda x: x['name'] == 'johndoe/forked_algo/1', content)[0]
self.assertEqual(algorithm['short_description'], '')
@@ -146,6 +172,79 @@ class AlgorithmsListRetrieval(AlgorithmsAPIBase):
self.assertEqual(algorithm['fork_of'], 'jackdoe/public_for_all/1')
self.assertEqual(algorithm['is_owner'], True)
self.assertEqual(algorithm['accessibility'], 'private')
+ self.assertEqual(algorithm['language'], 'python')
+
+ algorithm = filter(lambda x: x['name'] == 'jackdoe/binary_usable_by_one_user/1', content)[0]
+ self.assertEqual(algorithm['short_description'], '')
+ self.assertEqual(algorithm['version'], 1)
+ self.assertEqual(algorithm['previous_version'], None)
+ self.assertTrue(algorithm['fork_of'] is None)
+ self.assertEqual(algorithm['is_owner'], False)
+ self.assertEqual(algorithm['accessibility'], 'confidential')
+ self.assertEqual(algorithm['language'], 'cxx')
+
+ algorithm = filter(lambda x: x['name'] == 'jackdoe/binary_usable_by_one_team/1', content)[0]
+ self.assertEqual(algorithm['short_description'], '')
+ self.assertEqual(algorithm['version'], 1)
+ self.assertEqual(algorithm['previous_version'], None)
+ self.assertTrue(algorithm['fork_of'] is None)
+ self.assertEqual(algorithm['is_owner'], False)
+ self.assertEqual(algorithm['accessibility'], 'confidential')
+ self.assertEqual(algorithm['language'], 'cxx')
+
+ algorithm = filter(lambda x: x['name'] == 'jackdoe/binary_usable_by_teams/1', content)[0]
+ self.assertEqual(algorithm['short_description'], '')
+ self.assertEqual(algorithm['version'], 1)
+ self.assertEqual(algorithm['previous_version'], None)
+ self.assertTrue(algorithm['fork_of'] is None)
+ self.assertEqual(algorithm['is_owner'], False)
+ self.assertEqual(algorithm['accessibility'], 'confidential')
+ self.assertEqual(algorithm['language'], 'cxx')
+
+ algorithm = filter(lambda x: x['name'] == 'jackdoe/binary_public_for_all/1', content)[0]
+ self.assertEqual(algorithm['short_description'], '')
+ self.assertEqual(algorithm['version'], 1)
+ self.assertEqual(algorithm['previous_version'], None)
+ self.assertTrue(algorithm['fork_of'] is None)
+ self.assertEqual(algorithm['is_owner'], False)
+ self.assertEqual(algorithm['accessibility'], 'public')
+ self.assertEqual(algorithm['language'], 'cxx')
+
+ algorithm = filter(lambda x: x['name'] == 'jackdoe/binary_usable_by_all/1', content)[0]
+ self.assertEqual(algorithm['short_description'], '')
+ self.assertEqual(algorithm['version'], 1)
+ self.assertEqual(algorithm['previous_version'], None)
+ self.assertTrue(algorithm['fork_of'] is None)
+ self.assertEqual(algorithm['is_owner'], False)
+ self.assertEqual(algorithm['accessibility'], 'confidential')
+ self.assertEqual(algorithm['language'], 'cxx')
+
+ algorithm = filter(lambda x: x['name'] == 'jackdoe/binary_public_for_one_user/1', content)[0]
+ self.assertEqual(algorithm['short_description'], '')
+ self.assertEqual(algorithm['version'], 1)
+ self.assertEqual(algorithm['previous_version'], None)
+ self.assertTrue(algorithm['fork_of'] is None)
+ self.assertEqual(algorithm['is_owner'], False)
+ self.assertEqual(algorithm['accessibility'], 'confidential')
+ self.assertEqual(algorithm['language'], 'cxx')
+
+ algorithm = filter(lambda x: x['name'] == 'jackdoe/binary_public_for_one_team/1', content)[0]
+ self.assertEqual(algorithm['short_description'], '')
+ self.assertEqual(algorithm['version'], 1)
+ self.assertEqual(algorithm['previous_version'], None)
+ self.assertTrue(algorithm['fork_of'] is None)
+ self.assertEqual(algorithm['is_owner'], False)
+ self.assertEqual(algorithm['accessibility'], 'confidential')
+ self.assertEqual(algorithm['language'], 'cxx')
+
+ algorithm = filter(lambda x: x['name'] == 'jackdoe/binary_public_for_teams/1', content)[0]
+ self.assertEqual(algorithm['short_description'], '')
+ self.assertEqual(algorithm['version'], 1)
+ self.assertEqual(algorithm['previous_version'], None)
+ self.assertTrue(algorithm['fork_of'] is None)
+ self.assertEqual(algorithm['is_owner'], False)
+ self.assertEqual(algorithm['accessibility'], 'confidential')
+ self.assertEqual(algorithm['language'], 'cxx')
def test_own_algorithms(self):
@@ -157,7 +256,7 @@ class AlgorithmsListRetrieval(AlgorithmsAPIBase):
content = self.checkResponse(response, 200, content_type='application/json')
self.assertTrue(isinstance(content, list))
- self.assertEqual(len(content), 9)
+ self.assertEqual(len(content), 18)
self.assertEqual(len(filter(lambda x: x['name'] == 'jackdoe/usable_by_one_user/1', content)), 1)
self.assertEqual(len(filter(lambda x: x['name'] == 'jackdoe/usable_by_one_team/1', content)), 1)
@@ -169,43 +268,23 @@ class AlgorithmsListRetrieval(AlgorithmsAPIBase):
self.assertEqual(len(filter(lambda x: x['name'] == 'jackdoe/public_for_teams/1', content)), 1)
self.assertEqual(len(filter(lambda x: x['name'] == 'jackdoe/personal/1', content)), 1)
+ self.assertEqual(len(filter(lambda x: x['name'] == 'jackdoe/binary_usable_by_one_user/1', content)), 1)
+ self.assertEqual(len(filter(lambda x: x['name'] == 'jackdoe/binary_usable_by_one_team/1', content)), 1)
+ self.assertEqual(len(filter(lambda x: x['name'] == 'jackdoe/binary_usable_by_teams/1', content)), 1)
+ self.assertEqual(len(filter(lambda x: x['name'] == 'jackdoe/binary_usable_by_all/1', content)), 1)
+ self.assertEqual(len(filter(lambda x: x['name'] == 'jackdoe/binary_public_for_all/1', content)), 1)
+ self.assertEqual(len(filter(lambda x: x['name'] == 'jackdoe/binary_public_for_one_user/1', content)), 1)
+ self.assertEqual(len(filter(lambda x: x['name'] == 'jackdoe/binary_public_for_one_team/1', content)), 1)
+ self.assertEqual(len(filter(lambda x: x['name'] == 'jackdoe/binary_public_for_teams/1', content)), 1)
+ self.assertEqual(len(filter(lambda x: x['name'] == 'jackdoe/binary_personal/1', content)), 1)
algorithm = filter(lambda x: x['name'] == 'jackdoe/usable_by_one_user/1', content)[0]
self.assertEqual(algorithm['short_description'], '')
self.assertTrue(algorithm['fork_of'] is None)
- # self.assertEqual(len(algorithm['inputs']), 2)
- # self.assertEqual(len(algorithm['outputs']), 1)
-
- # input = algorithm['inputs'][0]
- # self.assertEqual(input['name'], 'a')
- # self.assertEqual(input['dataformat'], 'johndoe/single_integer/1')
-
- # input = algorithm['inputs'][1]
- # self.assertEqual(input['name'], 'b')
- # self.assertEqual(input['dataformat'], 'johndoe/single_integer/1')
-
- # output = algorithm['outputs'][0]
- # self.assertEqual(output['name'], 'sum')
- # self.assertEqual(output['dataformat'], 'johndoe/single_integer/1')
-
algorithm = filter(lambda x: x['name'] == 'jackdoe/public_for_all/1', content)[0]
self.assertEqual(algorithm['short_description'], '')
self.assertTrue(algorithm['fork_of'] is None)
- # self.assertEqual(len(algorithm['inputs']), 2)
- # self.assertEqual(len(algorithm['outputs']), 1)
-
- # input = algorithm['inputs'][0]
- # self.assertEqual(input['name'], 'a')
- # self.assertEqual(input['dataformat'], 'johndoe/single_integer/1')
-
- # input = algorithm['inputs'][1]
- # self.assertEqual(input['name'], 'b')
- # self.assertEqual(input['dataformat'], 'johndoe/single_integer/1')
-
- # output = algorithm['outputs'][0]
- # self.assertEqual(output['name'], 'sum')
- # self.assertEqual(output['dataformat'], 'johndoe/single_integer/1')
def test_other_user_algorithms(self):
@@ -217,55 +296,119 @@ class AlgorithmsListRetrieval(AlgorithmsAPIBase):
content = self.checkResponse(response, 200, content_type='application/json')
self.assertTrue(isinstance(content, list))
- self.assertEqual(len(content), 8)
+ self.assertEqual(len(content), 16)
algorithm = filter(lambda x: x['name'] == 'jackdoe/usable_by_one_user/1', content)[0]
self.assertEqual(algorithm['short_description'], '')
self.assertTrue(algorithm['fork_of'] is None)
self.assertEqual(algorithm['accessibility'], 'confidential')
self.assertFalse(algorithm['opensource'])
+ self.assertEqual(algorithm['language'], 'python')
algorithm = filter(lambda x: x['name'] == 'jackdoe/usable_by_one_team/1', content)[0]
self.assertEqual(algorithm['short_description'], '')
self.assertTrue(algorithm['fork_of'] is None)
self.assertEqual(algorithm['accessibility'], 'confidential')
self.assertFalse(algorithm['opensource'])
+ self.assertEqual(algorithm['language'], 'python')
algorithm = filter(lambda x: x['name'] == 'jackdoe/usable_by_teams/1', content)[0]
self.assertEqual(algorithm['short_description'], '')
self.assertTrue(algorithm['fork_of'] is None)
self.assertEqual(algorithm['accessibility'], 'confidential')
self.assertFalse(algorithm['opensource'])
+ self.assertEqual(algorithm['language'], 'python')
algorithm = filter(lambda x: x['name'] == 'jackdoe/usable_by_all/1', content)[0]
self.assertEqual(algorithm['short_description'], '')
self.assertTrue(algorithm['fork_of'] is None)
self.assertEqual(algorithm['accessibility'], 'confidential')
self.assertFalse(algorithm['opensource'])
+ self.assertEqual(algorithm['language'], 'python')
algorithm = filter(lambda x: x['name'] == 'jackdoe/public_for_one_user/1', content)[0]
self.assertEqual(algorithm['short_description'], '')
self.assertTrue(algorithm['fork_of'] is None)
self.assertEqual(algorithm['accessibility'], 'confidential')
self.assertTrue(algorithm['opensource'])
+ self.assertEqual(algorithm['language'], 'python')
algorithm = filter(lambda x: x['name'] == 'jackdoe/public_for_one_team/1', content)[0]
self.assertEqual(algorithm['short_description'], '')
self.assertTrue(algorithm['fork_of'] is None)
self.assertEqual(algorithm['accessibility'], 'confidential')
self.assertTrue(algorithm['opensource'])
+ self.assertEqual(algorithm['language'], 'python')
algorithm = filter(lambda x: x['name'] == 'jackdoe/public_for_teams/1', content)[0]
self.assertEqual(algorithm['short_description'], '')
self.assertTrue(algorithm['fork_of'] is None)
self.assertEqual(algorithm['accessibility'], 'confidential')
self.assertTrue(algorithm['opensource'])
+ self.assertEqual(algorithm['language'], 'python')
algorithm = filter(lambda x: x['name'] == 'jackdoe/public_for_all/1', content)[0]
self.assertEqual(algorithm['short_description'], '')
self.assertTrue(algorithm['fork_of'] is None)
self.assertEqual(algorithm['accessibility'], 'public')
self.assertTrue(algorithm['opensource'])
+ self.assertEqual(algorithm['language'], 'python')
+
+ algorithm = filter(lambda x: x['name'] == 'jackdoe/binary_usable_by_one_user/1', content)[0]
+ self.assertEqual(algorithm['short_description'], '')
+ self.assertTrue(algorithm['fork_of'] is None)
+ self.assertEqual(algorithm['accessibility'], 'confidential')
+ self.assertFalse(algorithm['opensource'])
+ self.assertEqual(algorithm['language'], 'cxx')
+
+ algorithm = filter(lambda x: x['name'] == 'jackdoe/binary_usable_by_one_team/1', content)[0]
+ self.assertEqual(algorithm['short_description'], '')
+ self.assertTrue(algorithm['fork_of'] is None)
+ self.assertEqual(algorithm['accessibility'], 'confidential')
+ self.assertFalse(algorithm['opensource'])
+ self.assertEqual(algorithm['language'], 'cxx')
+
+ algorithm = filter(lambda x: x['name'] == 'jackdoe/binary_usable_by_teams/1', content)[0]
+ self.assertEqual(algorithm['short_description'], '')
+ self.assertTrue(algorithm['fork_of'] is None)
+ self.assertEqual(algorithm['accessibility'], 'confidential')
+ self.assertFalse(algorithm['opensource'])
+ self.assertEqual(algorithm['language'], 'cxx')
+
+ algorithm = filter(lambda x: x['name'] == 'jackdoe/binary_usable_by_all/1', content)[0]
+ self.assertEqual(algorithm['short_description'], '')
+ self.assertTrue(algorithm['fork_of'] is None)
+ self.assertEqual(algorithm['accessibility'], 'confidential')
+ self.assertFalse(algorithm['opensource'])
+ self.assertEqual(algorithm['language'], 'cxx')
+
+ algorithm = filter(lambda x: x['name'] == 'jackdoe/binary_public_for_one_user/1', content)[0]
+ self.assertEqual(algorithm['short_description'], '')
+ self.assertTrue(algorithm['fork_of'] is None)
+ self.assertEqual(algorithm['accessibility'], 'confidential')
+ self.assertTrue(algorithm['opensource'])
+ self.assertEqual(algorithm['language'], 'cxx')
+
+ algorithm = filter(lambda x: x['name'] == 'jackdoe/binary_public_for_one_team/1', content)[0]
+ self.assertEqual(algorithm['short_description'], '')
+ self.assertTrue(algorithm['fork_of'] is None)
+ self.assertEqual(algorithm['accessibility'], 'confidential')
+ self.assertTrue(algorithm['opensource'])
+ self.assertEqual(algorithm['language'], 'cxx')
+
+ algorithm = filter(lambda x: x['name'] == 'jackdoe/binary_public_for_teams/1', content)[0]
+ self.assertEqual(algorithm['short_description'], '')
+ self.assertTrue(algorithm['fork_of'] is None)
+ self.assertEqual(algorithm['accessibility'], 'confidential')
+ self.assertTrue(algorithm['opensource'])
+ self.assertEqual(algorithm['language'], 'cxx')
+
+ algorithm = filter(lambda x: x['name'] == 'jackdoe/binary_public_for_all/1', content)[0]
+ self.assertEqual(algorithm['short_description'], '')
+ self.assertTrue(algorithm['fork_of'] is None)
+ self.assertEqual(algorithm['accessibility'], 'public')
+ self.assertTrue(algorithm['opensource'])
+ self.assertEqual(algorithm['language'], 'cxx')
def test_user_algorithms_for_anonymous_user(self):
@@ -275,13 +418,21 @@ class AlgorithmsListRetrieval(AlgorithmsAPIBase):
content = self.checkResponse(response, 200, content_type='application/json')
self.assertTrue(isinstance(content, list))
- self.assertEqual(len(content), 1)
+ self.assertEqual(len(content), 2)
algorithm = filter(lambda x: x['name'] == 'jackdoe/public_for_all/1', content)[0]
self.assertEqual(algorithm['short_description'], '')
self.assertTrue(algorithm['fork_of'] is None)
self.assertEqual(algorithm['accessibility'], 'public')
self.assertTrue(algorithm['opensource'])
+ self.assertEqual(algorithm['language'], 'python')
+
+ algorithm = filter(lambda x: x['name'] == 'jackdoe/binary_public_for_all/1', content)[0]
+ self.assertEqual(algorithm['short_description'], '')
+ self.assertTrue(algorithm['fork_of'] is None)
+ self.assertEqual(algorithm['accessibility'], 'public')
+ self.assertTrue(algorithm['opensource'])
+ self.assertEqual(algorithm['language'], 'cxx')
class AlgorithmsNameCheck(AlgorithmsAPIBase):
@@ -411,14 +562,17 @@ class AlgorithmCreation(AlgorithmsAPIBase):
"declaration (beyond white spaces): %r != %r" % \
(in_storage, expected)
- # set storage language so it can find the code
- storage.language = in_storage['language']
+ if in_storage['language'] == 'python':
+ self.assertTrue(algorithm.valid())
- in_storage = storage.code.try_load()
- expected = code
+ # set storage language so it can find the code
+ storage.language = in_storage['language']
- assert in_storage == expected, "There are differences on the " \
- "code: %r != %r" % (in_storage, expected)
+ in_storage = storage.code.try_load()
+ expected = code
+
+ assert in_storage == expected, "There are differences on the " \
+ "code: %r != %r" % (in_storage, expected)
def test_no_creation_for_anonymous_user(self):
@@ -769,6 +923,105 @@ class AlgorithmCreation(AlgorithmsAPIBase):
self.checkResponse(response, 400, content_type='application/json')
+ def test_cxx_algorithm(self):
+ self.client.login(username='jackdoe', password='1234')
+
+ response = self.client.post(self.url,
+ json.dumps({
+ 'name': 'valid-name1',
+ 'description': 'blah',
+ 'declaration': AlgorithmsAPIBase.CXX_DECLARATION,
+ }), content_type='application/json')
+
+ url = reverse('api_algorithms:object', args=['jackdoe', 'valid-name1', 1])
+ content = self.checkResponse(response, 201,
+ content_type='application/json',
+ location=url)
+
+ self.assertTrue(isinstance(content, dict))
+ self.assertEqual(content['name'], 'valid-name1')
+ self.assertTrue(content['url'].endswith(url))
+
+ try:
+ algorithm = Algorithm.objects.get(author__username='jackdoe', name='valid-name1')
+ except:
+ self.assertTrue(False)
+
+ self.assertEqual(algorithm.short_description, '')
+ self.assertEqual(algorithm.description, 'blah')
+ self.assertEqual(algorithm.language, Code.CXX)
+ self.assertFalse(algorithm.valid())
+ self.checkAlgorithm(algorithm, declaration=AlgorithmsAPIBase.CXX_DECLARATION, code=None)
+
+
+ def test_forked_cxx_algorithm(self):
+ self.client.login(username='jackdoe', password='1234')
+
+ response = self.client.post(self.url,
+ json.dumps({
+ 'name': 'valid-name1',
+ 'description': 'blah',
+ 'declaration': AlgorithmsAPIBase.CXX_DECLARATION,
+ 'fork_of': 'jackdoe/binary_personal/1',
+ }), content_type='application/json')
+
+ url = reverse('api_algorithms:object', args=['jackdoe', 'valid-name1', 1])
+ content = self.checkResponse(response, 201,
+ content_type='application/json',
+ location=url)
+
+ self.assertTrue(isinstance(content, dict))
+ self.assertEqual(content['name'], 'valid-name1')
+ self.assertTrue(content['url'].endswith(url))
+
+ try:
+ algorithm = Algorithm.objects.get(author__username='jackdoe', name='valid-name1')
+ except:
+ self.assertTrue(False)
+
+ self.assertEqual(algorithm.short_description, '')
+ self.assertEqual(algorithm.description, 'blah')
+ self.assertEqual(algorithm.language, Code.CXX)
+ self.assertTrue(algorithm.valid())
+ self.checkAlgorithm(algorithm, declaration=AlgorithmsAPIBase.CXX_DECLARATION, code=None)
+
+ self.assertEqual(algorithm.source_code, algorithm.fork_of.source_code)
+
+
+ def test_cxx_algorithm_version(self):
+ self.client.login(username='jackdoe', password='1234')
+
+ response = self.client.post(self.url,
+ json.dumps({
+ 'name': 'binary_personal',
+ 'description': 'blah',
+ 'declaration': AlgorithmsAPIBase.CXX_DECLARATION,
+ 'previous_version': 'jackdoe/binary_personal/1',
+ }), content_type='application/json')
+
+ url = reverse('api_algorithms:object', args=['jackdoe', 'binary_personal', 2])
+ content = self.checkResponse(response, 201,
+ content_type='application/json',
+ location=url)
+
+ self.assertTrue(isinstance(content, dict))
+ self.assertEqual(content['name'], 'binary_personal')
+ self.assertTrue(content['url'].endswith(url))
+
+ try:
+ algorithm = Algorithm.objects.get(author__username='jackdoe', name='binary_personal', version=2)
+ except:
+ self.assertTrue(False)
+
+ self.assertEqual(algorithm.short_description, '')
+ self.assertEqual(algorithm.description, 'blah')
+ self.assertEqual(algorithm.language, Code.CXX)
+ self.assertTrue(algorithm.valid())
+ self.checkAlgorithm(algorithm, declaration=AlgorithmsAPIBase.CXX_DECLARATION, code=None)
+
+ self.assertEqual(algorithm.source_code, algorithm.previous_version.source_code)
+
+
class AlgorithmUpdate(AlgorithmsAPIBase):
def setUp(self):
super(AlgorithmUpdate, self).setUp()
@@ -1438,6 +1691,90 @@ class AlgorithmUpdate(AlgorithmsAPIBase):
+class AlgorithmBinaryUpdate(AlgorithmsAPIBase):
+
+ def setUp(self):
+ super(AlgorithmBinaryUpdate, self).setUp()
+
+ self.url = reverse('api_algorithms:binary', args=['jackdoe', 'binary_personal', 1])
+ self.updated_binary_file = SimpleUploadedFile("algo.so", AlgorithmsAPIBase.BINARY_UPDATE,
+ content_type="application/octet-stream")
+
+ def test_no_update_for_anonymous_user(self):
+ response = self.client.post(self.url,
+ {
+ 'binary': self.updated_binary_file,
+ })
+
+ self.checkResponse(response, 403)
+
+
+ def test_fail_to_update_with_invalid_username(self):
+ self.client.login(username='jackdoe', password='1234')
+ url = reverse('api_algorithms:binary', args=['unknown', 'personal', 1])
+
+ response = self.client.post(url,
+ {
+ 'binary': self.updated_binary_file,
+ })
+
+ self.checkResponse(response, 404)
+
+
+ def test_fail_to_update_with_invalid_algorithm_name(self):
+ self.client.login(username='jackdoe', password='1234')
+ url = reverse('api_algorithms:binary', args=['jackdoe', 'unknown', 1])
+
+ response = self.client.post(url,
+ {
+ 'binary': self.updated_binary_file,
+ })
+
+ self.checkResponse(response, 404)
+
+
+ def test_no_update_without_content(self):
+ self.client.login(username='jackdoe', password='1234')
+ response = self.client.post(self.url,
+ {
+ })
+ self.checkResponse(response, 400)
+
+
+ def test_no_update_with_invalid_filename(self):
+ self.client.login(username='jackdoe', password='1234')
+ response = self.client.post(self.url,
+ {
+ 'unknown': self.updated_binary_file,
+ })
+ self.checkResponse(response, 400)
+
+
+ def test_no_update_with_invalid_file_content(self):
+ self.client.login(username='jackdoe', password='1234')
+ response = self.client.post(self.url,
+ {
+ 'binary': None,
+ })
+ self.checkResponse(response, 400)
+
+
+ def test_successfull_update(self):
+ self.client.login(username='jackdoe', password='1234')
+
+ response = self.client.post(self.url,
+ {
+ 'binary': self.updated_binary_file,
+ })
+
+ self.checkResponse(response, 204)
+
+ algorithm = Algorithm.objects.get(author__username='jackdoe', name='binary_personal', version=1)
+
+ self.assertEqual(algorithm.source_code, AlgorithmsAPIBase.BINARY_UPDATE)
+
+
+
class AlgorithmRetrieval(AlgorithmsAPIBase):
def test_no_retrieval_of_confidential_algorithm_for_anonymous_user(self):
@@ -1477,6 +1814,7 @@ class AlgorithmRetrieval(AlgorithmsAPIBase):
self.assertEqual(data['accessibility'], 'public')
self.assertTrue(data['opensource'])
+ self.assertEqual(data['language'], 'python')
self.assertEqual(data['description'], '')
self.assertFalse(data.has_key('is_owner'))
@@ -1490,6 +1828,7 @@ class AlgorithmRetrieval(AlgorithmsAPIBase):
self.assertEqual(data['accessibility'], 'confidential')
self.assertFalse(data['opensource'])
+ self.assertEqual(data['language'], 'python')
self.assertEqual(data['description'], '')
self.assertFalse(data.has_key('code'))
self.assertFalse(data.has_key('is_owner'))
@@ -1505,6 +1844,7 @@ class AlgorithmRetrieval(AlgorithmsAPIBase):
self.assertEqual(data['is_owner'], False)
self.assertEqual(data['accessibility'], 'public')
self.assertTrue(data['opensource'])
+ self.assertEqual(data['language'], 'python')
self.assertEqual(data['description'], '')
self.assertEqual(data['declaration'], AlgorithmsAPIBase.JSON_DECLARATION)
@@ -1521,6 +1861,7 @@ class AlgorithmRetrieval(AlgorithmsAPIBase):
self.assertEqual(data['is_owner'], False)
self.assertEqual(data['accessibility'], 'confidential')
self.assertFalse(data['opensource'])
+ self.assertEqual(data['language'], 'python')
self.assertEqual(data['description'], '')
self.assertFalse(data.has_key('code'))
@@ -1535,6 +1876,7 @@ class AlgorithmRetrieval(AlgorithmsAPIBase):
self.assertEqual(data['is_owner'], False)
self.assertEqual(data['accessibility'], 'confidential')
self.assertFalse(data['opensource'])
+ self.assertEqual(data['language'], 'python')
self.assertEqual(data['description'], '')
self.assertFalse(data.has_key('code'))
@@ -1549,6 +1891,7 @@ class AlgorithmRetrieval(AlgorithmsAPIBase):
self.assertEqual(data['is_owner'], False)
self.assertEqual(data['accessibility'], 'confidential')
self.assertTrue(data['opensource'])
+ self.assertEqual(data['language'], 'python')
self.assertEqual(data['description'], '')
self.assertEqual(data['declaration'], AlgorithmsAPIBase.JSON_DECLARATION)
@@ -1565,6 +1908,7 @@ class AlgorithmRetrieval(AlgorithmsAPIBase):
self.assertEqual(data['is_owner'], True)
self.assertEqual(data['accessibility'], 'public')
self.assertTrue(data['opensource'])
+ self.assertEqual(data['language'], 'python')
self.assertEqual(data['description'], '')
self.assertEqual(data['declaration'], AlgorithmsAPIBase.JSON_DECLARATION)
@@ -1586,6 +1930,7 @@ class AlgorithmRetrieval(AlgorithmsAPIBase):
self.assertEqual(data['is_owner'], True)
self.assertEqual(data['accessibility'], 'confidential')
self.assertFalse(data['opensource'])
+ self.assertEqual(data['language'], 'python')
self.assertEqual(data['description'], '')
self.assertEqual(data['declaration'], AlgorithmsAPIBase.JSON_DECLARATION)
@@ -1607,6 +1952,7 @@ class AlgorithmRetrieval(AlgorithmsAPIBase):
self.assertEqual(data['is_owner'], True)
self.assertEqual(data['accessibility'], 'confidential')
self.assertFalse(data['opensource'])
+ self.assertEqual(data['language'], 'python')
self.assertEqual(data['description'], '')
self.assertEqual(data['declaration'], AlgorithmsAPIBase.JSON_DECLARATION)
@@ -1628,6 +1974,7 @@ class AlgorithmRetrieval(AlgorithmsAPIBase):
self.assertEqual(data['is_owner'], True)
self.assertEqual(data['accessibility'], 'confidential')
self.assertFalse(data['opensource'])
+ self.assertEqual(data['language'], 'python')
self.assertEqual(data['description'], '')
self.assertEqual(data['declaration'], AlgorithmsAPIBase.JSON_DECLARATION)
@@ -1639,6 +1986,145 @@ class AlgorithmRetrieval(AlgorithmsAPIBase):
self.assertFalse(data['sharing'].has_key('usable_by'))
+ def test_successful_retrieval_of_binary_algorithm(self):
+ self.client.login(username='jackdoe', password='1234')
+
+ url = reverse('api_algorithms:object', args=['jackdoe', 'binary_personal'])
+ response = self.client.get(url)
+ data = self.checkResponse(response, 200, content_type='application/json')
+
+ self.assertEqual(data['is_owner'], True)
+ self.assertEqual(data['accessibility'], 'private')
+ self.assertFalse(data['opensource'])
+ self.assertEqual(data['language'], 'cxx')
+ self.assertEqual(data['description'], '')
+
+ self.assertEqual(data['declaration'], AlgorithmsAPIBase.JSON_CXX_DECLARATION)
+ self.assertFalse(data.has_key('code'))
+
+
+
+class AlgorithmBinaryRetrieval(AlgorithmsAPIBase):
+
+ def test_no_retrieval_of_confidential_algorithm_for_anonymous_user(self):
+ url = reverse('api_algorithms:binary', args=['jackdoe', 'binary_personal'])
+ response = self.client.get(url)
+ self.checkResponse(response, 404)
+
+
+ def test_fail_to_retrieve_with_invalid_username(self):
+ self.client.login(username='johndoe', password='1234')
+
+ url = reverse('api_algorithms:binary', args=['unknown', 'binary_personal'])
+ response = self.client.get(url)
+ self.checkResponse(response, 404)
+
+
+ def test_fail_to_retrieve_with_invalid_algorithm_name(self):
+ self.client.login(username='johndoe', password='1234')
+
+ url = reverse('api_algorithms:binary', args=['johndoe', 'unknown'])
+ response = self.client.get(url)
+ self.checkResponse(response, 404)
+
+
+ def test_no_retrieval_of_confidential_algorithm(self):
+ self.client.login(username='jackdoe', password='1234')
+
+ url = reverse('api_algorithms:binary', args=['johndoe', 'binary_personal'])
+ response = self.client.get(url)
+ self.checkResponse(response, 404)
+
+
+ def test_successful_retrieval_of_public_algorithm_for_anonymous_user(self):
+ url = reverse('api_algorithms:binary', args=['jackdoe', 'binary_public_for_all'])
+ response = self.client.get(url)
+ data = self.checkResponse(response, 200, content_type='application/octet-stream')
+
+ self.assertEqual(data, AlgorithmsAPIBase.BINARY)
+
+ def test_no_retrieval_of_usable_algorithm_for_anonymous_user(self):
+ url = reverse('api_algorithms:binary', args=['jackdoe', 'usable_by_all'])
+ response = self.client.get(url)
+ self.checkResponse(response, 404)
+
+
+ def test_successful_retrieval_of_public_algorithm(self):
+ self.client.login(username='johndoe', password='1234')
+
+ url = reverse('api_algorithms:binary', args=['jackdoe', 'binary_public_for_all'])
+ response = self.client.get(url)
+ data = self.checkResponse(response, 200, content_type='application/octet-stream')
+
+ self.assertEqual(data, AlgorithmsAPIBase.BINARY)
+
+
+ def test_no_retrieval_of_usable_algorithm(self):
+ self.client.login(username='johndoe', password='1234')
+
+ url = reverse('api_algorithms:binary', args=['jackdoe', 'binary_usable_by_one_user'])
+ response = self.client.get(url)
+ self.checkResponse(response, 404)
+
+
+ def test_no_retrieval_of_publicly_usable_algorithm(self):
+ self.client.login(username='johndoe', password='1234')
+
+ url = reverse('api_algorithms:binary', args=['jackdoe', 'binary_usable_by_all'])
+ response = self.client.get(url)
+ self.checkResponse(response, 404)
+
+
+ def test_successful_retrieval_of_confidential_algorithm(self):
+ self.client.login(username='johndoe', password='1234')
+
+ url = reverse('api_algorithms:binary', args=['jackdoe', 'binary_public_for_one_user'])
+ response = self.client.get(url)
+ data = self.checkResponse(response, 200, content_type='application/octet-stream')
+
+ self.assertEqual(data, AlgorithmsAPIBase.BINARY)
+
+
+ def test_successful_retrieval_of_own_public_algorithm(self):
+ self.client.login(username='jackdoe', password='1234')
+
+ url = reverse('api_algorithms:binary', args=['jackdoe', 'binary_public_for_all'])
+ response = self.client.get(url)
+ data = self.checkResponse(response, 200, content_type='application/octet-stream')
+
+ self.assertEqual(data, AlgorithmsAPIBase.BINARY)
+
+
+ def test_successful_retrieval_of_own_confidential_algorithm(self):
+ self.client.login(username='jackdoe', password='1234')
+
+ url = reverse('api_algorithms:binary', args=['jackdoe', 'binary_personal'])
+ response = self.client.get(url)
+ data = self.checkResponse(response, 200, content_type='application/octet-stream')
+
+ self.assertEqual(data, AlgorithmsAPIBase.BINARY)
+
+
+ def test_successful_retrieval_of_own_usable_algorithm(self):
+ self.client.login(username='jackdoe', password='1234')
+
+ url = reverse('api_algorithms:binary', args=['jackdoe', 'binary_usable_by_all'])
+ response = self.client.get(url)
+ data = self.checkResponse(response, 200, content_type='application/octet-stream')
+
+ self.assertEqual(data, AlgorithmsAPIBase.BINARY)
+
+
+ def test_successful_retrieval_of_own_shared_algorithm(self):
+ self.client.login(username='jackdoe', password='1234')
+
+ url = reverse('api_algorithms:binary', args=['jackdoe', 'binary_public_for_one_user'])
+ response = self.client.get(url)
+ data = self.checkResponse(response, 200, content_type='application/octet-stream')
+
+ self.assertEqual(data, AlgorithmsAPIBase.BINARY)
+
+
class AlgorithmDeletion(AlgorithmsAPIBase):
diff --git a/beat/web/algorithms/views.py b/beat/web/algorithms/views.py
old mode 100644
new mode 100755
index 70839094f22e5d836b001cab7fac779be0aaa642..a42fd4699a1395964f3b43bf31379653193b4d34
--- a/beat/web/algorithms/views.py
+++ b/beat/web/algorithms/views.py
@@ -75,12 +75,18 @@ def create(request, name=None):
description = previous_version.description
- parameters['algorithm_version'] = previous_version.version + 1
- parameters['source_code'] = previous_version.source_code_file.read()
- parameters['declaration'] = previous_version.declaration_file.read().replace('\n', '')
- parameters['short_description'] = previous_version.short_description
- parameters['description'] = description.replace('\n', '\\n')
- parameters['html_description'] = restructuredtext(description).replace('\n', '')
+ parameters['algorithm_version'] = previous_version.version + 1
+ parameters['declaration'] = previous_version.declaration_file.read().replace('\n', '')
+ parameters['short_description'] = previous_version.short_description
+ parameters['description'] = description.replace('\n', '\\n')
+ parameters['html_description'] = restructuredtext(description).replace('\n', '')
+ parameters['algorithm_language'] = previous_version.json_language
+ parameters['algorithm_language_name'] = previous_version.language_fullname()
+ parameters['binary'] = previous_version.is_binary()
+ parameters['new_version'] = True
+
+ if not previous_version.is_binary():
+ parameters['source_code'] = previous_version.source_code_file.read()
else:
parameters['source_code'] = prototypes.binary_load('algorithm.py') \
.replace('\n\n # TODO: Implement this algorithm\n\n',
@@ -118,21 +124,26 @@ def fork(request, author, name, version):
description = fork_of.description
- parameters = {'original_author': author,
- 'algorithm_author': request.user.username,
- 'algorithm_name': name,
- 'algorithm_version': fork_of.version,
- 'source_code': fork_of.source_code_file.read(),
- 'declaration': fork_of.declaration_file.read().replace('\n', ''),
- 'short_description': fork_of.short_description,
- 'description': description.replace('\n', '\\n'),
- 'html_description': restructuredtext(description).replace('\n', ''),
- 'messages': Messages,
- 'fork_of': fork_of,
- 'edition': False,
- 'plot_account': settings.PLOT_ACCOUNT,
+ parameters = {'original_author': author,
+ 'algorithm_author': request.user.username,
+ 'algorithm_name': name,
+ 'algorithm_version': fork_of.version,
+ 'algorithm_language': fork_of.json_language,
+ 'algorithm_language_name': fork_of.language_fullname(),
+ 'declaration': fork_of.declaration_file.read().replace('\n', ''),
+ 'short_description': fork_of.short_description,
+ 'description': description.replace('\n', '\\n'),
+ 'html_description': restructuredtext(description).replace('\n', ''),
+ 'messages': Messages,
+ 'fork_of': fork_of,
+ 'edition': False,
+ 'binary': fork_of.is_binary(),
+ 'plot_account': settings.PLOT_ACCOUNT,
}
+ if not fork_of.is_binary():
+ parameters['source_code'] = fork_of.source_code_file.read()
+
return render_to_response('algorithms/edition.html',
parameters,
context_instance=RequestContext(request))
@@ -164,19 +175,24 @@ def edit(request, author, name, version):
description = algorithm.description
- parameters = {'algorithm_author': request.user.username,
- 'algorithm_name': name,
- 'algorithm_version': algorithm.version,
- 'source_code': algorithm.source_code_file.read(),
- 'declaration': algorithm.declaration_file.read().replace('\n', ''),
- 'short_description': algorithm.short_description,
- 'description': description.replace('\n', '\\n'),
- 'html_description': restructuredtext(description).replace('\n', ''),
- 'messages': Messages,
- 'edition': True,
- 'plot_account': settings.PLOT_ACCOUNT,
+ parameters = {'algorithm_author': request.user.username,
+ 'algorithm_name': name,
+ 'algorithm_version': algorithm.version,
+ 'algorithm_language': algorithm.json_language,
+ 'algorithm_language_name': algorithm.language_fullname(),
+ 'declaration': algorithm.declaration_file.read().replace('\n', ''),
+ 'short_description': algorithm.short_description,
+ 'description': description.replace('\n', '\\n'),
+ 'html_description': restructuredtext(description).replace('\n', ''),
+ 'messages': Messages,
+ 'edition': True,
+ 'binary': algorithm.is_binary(),
+ 'plot_account': settings.PLOT_ACCOUNT,
}
+ if not algorithm.is_binary():
+ parameters['source_code'] = algorithm.source_code_file.read()
+
return render_to_response('algorithms/edition.html',
parameters,
context_instance=RequestContext(request))
diff --git a/beat/web/backend/admin.py b/beat/web/backend/admin.py
old mode 100644
new mode 100755
index 8658a41a4fae0995530e881fe85233e4f3664233..3a113b9e26426d0d6b92954b814dfe005fcad630
--- a/beat/web/backend/admin.py
+++ b/beat/web/backend/admin.py
@@ -29,6 +29,7 @@ from django.contrib import admin
from django import forms
from .models import Environment as EnvironmentModel
+from .models import EnvironmentLanguage as EnvironmentLanguageModel
from .models import Worker as WorkerModel
from .models import Queue as QueueModel
from .models import Slot as SlotModel
@@ -64,6 +65,10 @@ class EnvironmentModelForm(forms.ModelForm):
}
+class EnvironmentLanguageInline(admin.TabularInline):
+ model = EnvironmentLanguageModel
+
+
class Environment(admin.ModelAdmin):
list_display = (
@@ -87,6 +92,10 @@ class Environment(admin.ModelAdmin):
'name',
)
+ inlines = [
+ EnvironmentLanguageInline
+ ]
+
form = EnvironmentModelForm
filter_horizontal = [
diff --git a/beat/web/backend/api.py b/beat/web/backend/api.py
old mode 100644
new mode 100755
index 258472e8c79f2a6ef52deb04bcdc01e5226f3528..f0757c3b6973d8c3c4e22e8e1d50845e98f377bc
--- a/beat/web/backend/api.py
+++ b/beat/web/backend/api.py
@@ -30,6 +30,7 @@ from rest_framework.response import Response
from rest_framework import permissions
from .models import Environment
+from ..code.models import Code
@api_view(['GET'])
@@ -68,6 +69,7 @@ def accessible_environments_list(request):
'short_description': environment.short_description,
'queues': queues,
'accessibility': accessibility,
+ 'languages': [ Code.language_identifier(x.language) for x in environment.languages.iterator() ],
})
return Response(result)
diff --git a/beat/web/backend/management/commands/qsetup.py b/beat/web/backend/management/commands/qsetup.py
old mode 100644
new mode 100755
index 080d869647301e62cc08f5b61cb68893a66aa140..f903d45240bf62d11a3dc97edec4426fe2c6b575
--- a/beat/web/backend/management/commands/qsetup.py
+++ b/beat/web/backend/management/commands/qsetup.py
@@ -43,7 +43,9 @@ import socket
CORES = psutil.cpu_count()
RAM = psutil.virtual_memory().total/(1024*1024)
ENVIRONMENT = {'name': 'environment', 'version': '1'}
+CXX_ENVIRONMENT = {'name': 'cxx_environment', 'version': '1'}
ENVKEY = '%(name)s (%(version)s)' % ENVIRONMENT
+CXX_ENVKEY = '%(name)s (%(version)s)' % CXX_ENVIRONMENT
HOSTNAME = socket.gethostname()
DEFAULT_CONFIGURATION = {
@@ -53,7 +55,7 @@ DEFAULT_CONFIGURATION = {
"time-limit": 1440, #1 day
"cores-per-slot": 1,
"max-slots-per-user": CORES,
- "environments": [ENVKEY],
+ "environments": [CXX_ENVKEY, ENVKEY],
"slots": {
HOSTNAME: {
"quantity": CORES,
@@ -69,10 +71,18 @@ DEFAULT_CONFIGURATION = {
ENVKEY: {
"name": ENVIRONMENT['name'],
"version": ENVIRONMENT['version'],
+ "languages": ['python'],
"short_description": "Local python interpreter",
"description": "Automatically generated local python " \
"interpreter environment",
},
+ CXX_ENVKEY: {
+ "name": CXX_ENVIRONMENT['name'],
+ "version": CXX_ENVIRONMENT['version'],
+ "languages": ['cxx'],
+ "short_description": "C++ backend",
+ "description": "C++ backend running in a docker container",
+ },
},
"workers": {
HOSTNAME: {
diff --git a/beat/web/backend/migrations/0004_environmentlanguage.py b/beat/web/backend/migrations/0004_environmentlanguage.py
new file mode 100644
index 0000000000000000000000000000000000000000..53a3594d3c4ab75314e61f0e0e21aaeecfcb9838
--- /dev/null
+++ b/beat/web/backend/migrations/0004_environmentlanguage.py
@@ -0,0 +1,34 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.11 on 2016-11-23 12:18
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+def add_default_language(apps, schema_editor):
+ Environment = apps.get_model("backend", "Environment")
+ EnvironmentLanguage = apps.get_model("backend", "EnvironmentLanguage")
+
+ for env in Environment.objects.all():
+ lang = EnvironmentLanguage(language='P', environment=env)
+ lang.save()
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('backend', '0003_remove_result_syserr'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='EnvironmentLanguage',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('language', models.CharField(choices=[(b'U', b'Unknown'), (b'C', b'Cxx'), (b'M', b'Matlab'), (b'P', b'Python'), (b'R', b'R')], default=b'P', max_length=1)),
+ ('environment', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='languages', to='backend.Environment')),
+ ],
+ ),
+ migrations.RunPython(add_default_language, migrations.RunPython.noop),
+ ]
diff --git a/beat/web/backend/models.py b/beat/web/backend/models.py
old mode 100644
new mode 100755
index d896979b1209371e657a6e312ad2f149bdde8abf..9a4d702ecc0472cbac66967ac1aab5773ef8ab2f
--- a/beat/web/backend/models.py
+++ b/beat/web/backend/models.py
@@ -52,7 +52,9 @@ from guardian.shortcuts import get_perms
import beat.core.stats
import beat.core.data
import beat.core.execution
+from beat.core.dock import Host
+from ..code.models import Code
from ..common.models import Shareable, ShareableManager
from ..common.texts import Messages
from ..statistics.utils import updateStatistics
@@ -164,6 +166,16 @@ class Environment(Shareable):
)
+class EnvironmentLanguage(models.Model):
+
+ environment = models.ForeignKey(Environment,
+ related_name='languages'
+ )
+
+ language = models.CharField(max_length=1, choices=Code.CODE_LANGUAGE,
+ default=Code.PYTHON)
+
+
#----------------------------------------------------------
@@ -400,7 +412,7 @@ class Worker(models.Model):
- def work(self, environments, cpulimit, process):
+ def work(self, environments, process):
'''Launches user code on isolated processes
This function is supposed to be called asynchronously, by a
@@ -418,15 +430,10 @@ class Worker(models.Model):
environments (dict): A dictionary containing installed
environments, their description and execute-file paths.
- cpulimit (str): The path to the ``cpulimit`` program to use for
- limiting the user code in CPU usage. If set to ``None``, then
- don't use it, even if the select user queue has limits.
-
process (str): The path to the ``process.py`` program to use for
running the user code on isolated processes.
'''
- from .utils import pick_execute
# refresh state from database and update state if required
self.refresh_from_db()
@@ -453,9 +460,10 @@ class Worker(models.Model):
# cmdline base argument
cmdline = [process]
- if cpulimit is not None: cmdline += ['--cpulimit=%s' % cpulimit]
- if settings.DEBUG: cmdline += ['-vv']
- else: cmdline += ['-v']
+ if settings.DEBUG:
+ cmdline += ['-vv']
+ else:
+ cmdline += ['-v']
# start newly assigned job splits
with transaction.atomic():
@@ -463,31 +471,12 @@ class Worker(models.Model):
status=Job.QUEUED, start_date__isnull=True,
process_id__isnull=True)
for split in splits:
- execute = pick_execute(split, environments)
- if execute is None:
- message = "Environment `%s' is not available for split " \
- "%d/%d running at worker `%s', for block `%s' of " \
- "experiment `%s': %s" % \
- (split.job.block.environment,
- split.split_index+1,
- split.job.block.required_slots,
- self,
- split.job.block.name,
- split.job.block.experiment.fullname(),
- "Available environments are `%s'" % \
- '|'.join(environments.keys()),
- )
- logger.error(message)
- split.end(Result(status=1,
- usrerr=settings.DEFAULT_USER_ERROR))
- continue
-
# if we get to this point, then we launch the user process
# -> see settings.WORKER_DETACH_CHILDREN for more info
kwargs = dict()
if settings.WORKER_DETACH_CHILDREN:
kwargs['preexec_fn'] = os.setpgrp
- subprocess.Popen(cmdline + [execute, str(split.pk)], **kwargs)
+ subprocess.Popen(cmdline + [str(split.pk)], **kwargs)
split.status = Job.PROCESSING #avoids re-running
split.save()
@@ -1129,6 +1118,9 @@ class JobSplit(models.Model):
process_id = models.PositiveIntegerField(null=True)
+ host = None
+
+
class Meta:
unique_together = ('job', 'split_index')
@@ -1373,7 +1365,7 @@ class JobSplit(models.Model):
self.end(result)
- def process(self, execute, cpulimit=None, cache=settings.CACHE_ROOT):
+ def process(self, cache=settings.CACHE_ROOT):
'''Process assigned job splits using beat.core
This task executes the user algorithm on a subprocess. It also serves
@@ -1383,26 +1375,9 @@ class JobSplit(models.Model):
set. Otherwise, it takes care of a subset of the input data that is
synchronised with this block, determined by ``split_index``.
- Two processes are spawned from the current work process:
-
- * The process for executing the user code
- * A process to limit the CPU usage (with ``cpulimit``), if these
- conditions are respected:
-
- 1. The program ``cpulimit`` is available on the current machine
- 2. The configuration requests a CPU usage greater than 0 (``nb_cores
- > 0``). (N.B.: a value of zero means not to limit on CPU).
-
Parameters:
- execute (str): The path to the ``execute`` program to use for running
- the user code associated with this job split.
-
- cpulimit (str, Optional): The path to the ``cpulimit`` program to use
- for limiting the user code in CPU usage. If not set, then don't use
- it, even if the select user queue has limits.
-
cache (str, Optional): The path leading to the root of the cache to
use for this run. If not set, use the global default at
``settings.CACHE_ROOT``.
@@ -1412,6 +1387,8 @@ class JobSplit(models.Model):
logger.info("Starting to process split `%s' (pid=%d)...", self,
os.getpid())
+ self.executor = None
+
config = simplejson.loads(self.job.block.command)
# setup range if necessary
@@ -1435,12 +1412,16 @@ class JobSplit(models.Model):
try:
- executor = beat.core.execution.Executor(settings.PREFIX, config,
+ if JobSplit.host is None:
+ JobSplit.host = Host()
+ JobSplit.host.setup(raise_on_errors=not(getattr(settings, 'TEST_CONFIGURATION', False)))
+
+ self.executor = beat.core.execution.Executor(settings.PREFIX, config,
cache)
- if not executor.valid:
+ if not self.executor.valid:
err = ''
- for e in executor.errors: err += ' * %s\n' % e
+ for e in self.executor.errors: err += ' * %s\n' % e
message = "Failed to load execution information for split " \
"%d/%d running at worker `%s', for block `%s' of " \
"experiment `%s': %s" % (self.split_index+1,
@@ -1450,24 +1431,17 @@ class JobSplit(models.Model):
raise RuntimeError(message)
queue = self.job.block.queue
- nb_cores = queue.cores_per_slot
- if (nb_cores > 0) and (cpulimit is None):
- logger.warn("Job requires limiting CPU usage to %g (cores), " \
- "but you have not set the path to the program " \
- "`cpulimit'. Continuing without CPU limiting...", nb_cores)
- nb_cores = 0
logger.info("Running `%s' on worker request",
- executor.algorithm.name)
+ self.executor.algorithm.name)
# n.b.: with executor may crash on the database view setup
- with executor:
+ with self.executor:
self.start()
- result = executor.process(
- execute_path=execute,
+ result = self.executor.process(
+ JobSplit.host,
virtual_memory_in_megabytes=queue.memory_limit,
- max_cpu_percent=int(100*float(nb_cores)), #allows for 150%
- cpulimit_path=cpulimit,
+ max_cpu_percent=int(100*float(queue.cores_per_slot)), #allows for 150%
timeout_in_minutes=queue.time_limit,
daemon=0,
)
@@ -1501,3 +1475,5 @@ class JobSplit(models.Model):
logger.error("Split `%s' (pid=%d) ended with an error: %s",
self, os.getpid(), traceback.format_exc())
self.try_end(Result(status=1, usrerr=settings.DEFAULT_USER_ERROR))
+
+ self.executor = None
diff --git a/beat/web/backend/static/backend/js/models.js b/beat/web/backend/static/backend/js/models.js
index 18deadbe8d5cd687d8d9649b799603e25fb98082..33ef15c7411938b9598826eb5d399cd95e86d4c1 100644
--- a/beat/web/backend/static/backend/js/models.js
+++ b/beat/web/backend/static/backend/js/models.js
@@ -56,6 +56,7 @@ beat.backend.models.EnvironmentsList = function(declaration)
var environment = {
name: declaration[i].name,
version: declaration[i].version,
+ languages: declaration[i].languages,
queues: [],
};
@@ -127,6 +128,51 @@ beat.backend.models.EnvironmentsList.prototype.contains = function(name, version
}
+//----------------------------------------------------------------------------------------
+// Retrieve a list of environments supporting a specific language
+//----------------------------------------------------------------------------------------
+beat.backend.models.EnvironmentsList.prototype.filter = function(language)
+{
+ var result = new beat.backend.models.EnvironmentsList([]);
+
+ for (var i = 0; i < this.data.length; i++)
+ {
+ var environment = this.data[i];
+
+ if (environment.languages.indexOf(language) >= 0)
+ result.data.push(JSON.parse(JSON.stringify(environment)));
+ }
+
+ result.length = result.data.length;
+
+ return result;
+}
+
+
+//----------------------------------------------------------------------------------------
+// Retrieve the list of supported languages across all the environments
+//----------------------------------------------------------------------------------------
+beat.backend.models.EnvironmentsList.prototype.languages = function()
+{
+ var languages = [];
+
+ for (var i = 0; i < this.data.length; i++)
+ {
+ var environment = this.data[i];
+
+ for (var j = 0; j < environment.languages.length; j++)
+ {
+ var language = environment.languages[j];
+
+ if (languages.indexOf(language) < 0)
+ languages.push(language);
+ }
+ }
+
+ return languages;
+}
+
+
//----------------------------------------------------------------------------------------
// Returns an iterator over the environments
//----------------------------------------------------------------------------------------
diff --git a/beat/web/backend/tests.py b/beat/web/backend/tests.py
old mode 100644
new mode 100755
index 977a2384276731419b4e2343b3fc4575142f1d0b..e4cfea7fdb4665084f15531839fe800532202510
--- a/beat/web/backend/tests.py
+++ b/beat/web/backend/tests.py
@@ -31,6 +31,7 @@ import time
import shutil
import tempfile
import collections
+import time
from django.conf import settings
from django.core.urlresolvers import reverse
@@ -133,6 +134,7 @@ QUEUES_WITHOUT_PRIORITY = {
"version": '1',
"short_description": "Test",
"description": "Test environment",
+ "languages": "python",
},
},
}
@@ -222,6 +224,7 @@ PRIORITY_QUEUES = {
"version": '1',
"short_description": "Test",
"description": "Test environment",
+ "languages": "python",
},
},
}
@@ -380,7 +383,7 @@ class BaseBackendTestCase(TestCase):
setup_backend(qsetup.DEFAULT_CONFIGURATION)
Worker.objects.update(active=True)
- env = Environment.objects.first()
+ env = Environment.objects.get(name='environment')
queue = Queue.objects.first()
template_data = dict(
@@ -1959,7 +1962,7 @@ class SchedulingPriority(BaseBackendTestCase):
q1 = Queue.objects.get(name='q1')
q2 = Queue.objects.get(name='q2')
- env = Environment.objects.get()
+ env = Environment.objects.get(name='environment')
# reset queue and environment to new backend configuration
self.set_globals(xp, q1, env)
@@ -2006,7 +2009,7 @@ class SchedulingPriority(BaseBackendTestCase):
q1 = Queue.objects.get(name='q1')
q4 = Queue.objects.get(name='q4')
- env = Environment.objects.get()
+ env = Environment.objects.get(name='environment')
# reset queue and environment to new backend configuration
self.set_globals(xp, q1, env)
@@ -2046,7 +2049,7 @@ class SchedulingPriority(BaseBackendTestCase):
Worker.objects.update(active=True)
q1 = Queue.objects.get(name='q1')
- env = Environment.objects.get()
+ env = Environment.objects.get(name='environment')
fullname = 'user/user/single/1/single'
xp = Experiment.objects.get(name=fullname.split(os.sep)[-1])
@@ -2093,13 +2096,9 @@ class Working(BaseBackendTestCase):
def setUp(self):
- from beat.core.async import resolve_cpulimit_path
- self.cpulimit = resolve_cpulimit_path(None)
-
from . import utils
self.process = utils.resolve_process_path()
self.environments = utils.find_environments(None)
- self.env1_execute = self.environments['environment (1)']['execute']
if not os.path.exists(settings.CACHE_ROOT):
os.makedirs(settings.CACHE_ROOT)
@@ -2150,7 +2149,7 @@ class Working(BaseBackendTestCase):
self.assertEqual(worker.available_cores(), qsetup.CORES-1)
# actually runs the job (blocking)
- split.process(self.env1_execute, self.cpulimit)
+ split.process()
# at this point, job should have been successful
xp.refresh_from_db()
@@ -2188,7 +2187,7 @@ class Working(BaseBackendTestCase):
self.assertEqual(worker.available_cores(), qsetup.CORES-1)
# actually runs the job (blocking)
- split.process(self.env1_execute, self.cpulimit)
+ split.process()
# checks the number of statistics objects has increased by 1
self.assertEqual(HourlyStatistics.objects.count(), current_stats + 1)
@@ -2240,7 +2239,7 @@ class Working(BaseBackendTestCase):
self.assertEqual(worker.available_cores(), qsetup.CORES-1)
# actually runs the job (blocking)
- split.process(self.env1_execute, self.cpulimit)
+ split.process()
# at this point, job should have failed
xp.refresh_from_db()
@@ -2259,7 +2258,6 @@ class Working(BaseBackendTestCase):
assert block.linear_execution_time() > 0.0
assert block.queuing_time() > 0.0
assert block.stdout() == ''
- assert block.stderr() == ''
assert block.error_report().find('Error') != -1
# assert we have no database traces after the block is done
@@ -2300,7 +2298,7 @@ class Working(BaseBackendTestCase):
self.assertEqual(worker.available_cores(), qsetup.CORES-1)
# actually runs the job (blocking)
- split.process(self.env1_execute, self.cpulimit)
+ split.process()
# at this point, job should have been successful
xp.refresh_from_db()
@@ -2338,7 +2336,7 @@ class Working(BaseBackendTestCase):
self.assertEqual(worker.available_cores(), qsetup.CORES-1)
# actually runs the job (blocking)
- split.process(self.env1_execute, self.cpulimit)
+ split.process()
# checks the number of statistics objects has increased by 1
self.assertEqual(HourlyStatistics.objects.count(), current_stats + 1)
@@ -2399,7 +2397,7 @@ class Working(BaseBackendTestCase):
self.assertEqual(worker.available_cores(), qsetup.CORES-1)
# actually runs the job (blocking)
- split.process(self.env1_execute, self.cpulimit)
+ split.process()
# at this point, job should have been successful
xp.refresh_from_db()
@@ -2448,7 +2446,7 @@ class Working(BaseBackendTestCase):
self.assertEqual(worker.available_cores(), qsetup.CORES-1)
# actually runs the job (blocking)
- split.process(self.env1_execute, self.cpulimit)
+ split.process()
# checks the number of statistics objects has increased by 1
self.assertEqual(HourlyStatistics.objects.count(), current_stats + 1)
@@ -2510,7 +2508,7 @@ class Working(BaseBackendTestCase):
self.assertEqual(worker.available_cores(), qsetup.CORES-1)
# actually runs the job (blocking)
- split.process(self.env1_execute, self.cpulimit)
+ split.process()
# at this point, job should have been successful
xp.refresh_from_db()
@@ -2548,9 +2546,6 @@ class WorkingExternally(TransactionTestCase):
def setUp(self):
- from beat.core.async import resolve_cpulimit_path
- self.cpulimit = resolve_cpulimit_path(None)
-
from . import utils
self.process = utils.resolve_process_path()
self.environments = utils.find_environments(None)
@@ -2565,7 +2560,7 @@ class WorkingExternally(TransactionTestCase):
setup_backend(qsetup.DEFAULT_CONFIGURATION)
Worker.objects.update(active=True)
- env = Environment.objects.first()
+ env = Environment.objects.get(name='environment')
queue = Queue.objects.first()
template_data = dict(
@@ -2621,7 +2616,7 @@ class WorkingExternally(TransactionTestCase):
self.assertEqual(worker.available_cores(), qsetup.CORES-1)
# actually runs the job (non-blocking)
- worker.work(self.environments, self.cpulimit, self.process)
+ worker.work(self.environments, self.process)
def condition():
xp.refresh_from_db()
@@ -2665,7 +2660,7 @@ class WorkingExternally(TransactionTestCase):
self.assertEqual(worker.available_cores(), qsetup.CORES-1)
# actually runs the job (non-blocking)
- worker.work(self.environments, self.cpulimit, self.process)
+ worker.work(self.environments, self.process)
def condition():
xp.refresh_from_db()
@@ -2723,20 +2718,24 @@ class WorkingExternally(TransactionTestCase):
self.assertEqual(worker.available_cores(), qsetup.CORES-1)
# actually runs the job (non-blocking)
- worker.work(self.environments, self.cpulimit, self.process)
+ worker.work(self.environments, self.process)
def condition():
xp.refresh_from_db()
return xp.status == Experiment.RUNNING
+
_sleep(20, condition)
+ # Just to be sure that the docker container really started
+ time.sleep(3)
+
# cancels the experiment
xp.cancel()
split.refresh_from_db()
self.assertEqual(split.status, Job.CANCEL)
# launch another working cycle to kill the process
- worker.work(self.environments, self.cpulimit, self.process)
+ worker.work(self.environments, self.process)
def condition():
xp.refresh_from_db()
diff --git a/beat/web/backend/utils.py b/beat/web/backend/utils.py
old mode 100644
new mode 100755
index bda58d941fe8662d8012e03e03f3f9588bcf0f43..ff220f74dd037318d6450edfbc0d5864f220c97c
--- a/beat/web/backend/utils.py
+++ b/beat/web/backend/utils.py
@@ -32,19 +32,22 @@ import sys
import fnmatch
import glob
import time
+import distutils.spawn
import logging
logger = logging.getLogger(__name__)
import psutil
+from django.conf import settings
from django.db import transaction
from django.contrib.auth.models import Group
from guardian.shortcuts import assign_perm
+from ..code.models import Code
from ..common.models import Shareable
from ..experiments.models import CachedFile, Block, Experiment
-from .models import Queue, Worker, Job, Environment, Slot
+from .models import Queue, Worker, Job, Environment, EnvironmentLanguage, Slot
def cleanup_cache(path, age_in_minutes=0, delete=False):
@@ -218,6 +221,13 @@ def setup_backend(d):
logger.info("Creating `%s'...", env)
env.save()
+ for language in attrs['languages']:
+ lang = EnvironmentLanguage(
+ language=Code.language_db(language),
+ environment=env
+ )
+ lang.save()
+
# 8.1 Create new workers
config_workers = set(d['workers'].keys())
current_workers = set(Worker.objects.values_list('name', flat=True))
@@ -337,15 +347,20 @@ def setup_backend(d):
def dump_backend():
'''Returns a dictionary that represents the current backend configuration'''
+ environments = {}
+ for env in Environment.objects.all():
+ environments[str(env)] = env.as_dict()
+ environments[str(env)]['languages'] = [ Code.language_identifier(x.language) for x in env.languages.iterator() ]
+
return dict(
queues=dict([(k.name, k.as_dict()) for k in Queue.objects.all()]),
- environments=dict([(str(k), k.as_dict()) for k in Environment.objects.all()]),
+ environments=environments,
workers=dict([(k.name, k.as_dict()) for k in Worker.objects.all()]),
)
def resolve_process_path():
- '''Returns the path to cpulimit'''
+ '''Returns the path to process.py'''
basedir = os.path.dirname(os.path.realpath(sys.argv[0]))
r = os.path.join(basedir, 'process')
@@ -386,26 +401,8 @@ def find_environments(paths=None):
'''
- from beat.core.execution import discover_environments
-
- if paths is not None:
- logger.debug("Search for environments at `%s'", os.pathsep.join(paths))
- retval = discover_environments(paths)
- logger.debug("Found %d environment(s)", len(retval))
- return retval
-
- else:
- import pkg_resources
- path = pkg_resources.resource_filename(__name__, 'environments')
- logger.debug("Search for environments at `%s'", path)
- retval = discover_environments([path])
- logger.debug("Found %d environment(s)", len(retval))
- return retval
-
-
-def pick_execute(split, environments):
- """Resolves the path to the ``execute`` program to use for the split"""
+ from beat.core.dock import Host
- # Check we have a compatible environment to execute the user algorithm
- envinfo = environments.get(split.job.block.environment.fullname())
- return envinfo['execute'] if envinfo else None
+ host = Host()
+ host.setup(raise_on_errors=not(getattr(settings, 'TEST_CONFIGURATION', False)))
+ return host.environments
diff --git a/beat/web/backend/views.py b/beat/web/backend/views.py
old mode 100644
new mode 100755
index e954598b6b5fc2e962ac70a0db862681ceab4c26..b3c953c6c3f26dda4274f36f1c80e30f38b2b5d1
--- a/beat/web/backend/views.py
+++ b/beat/web/backend/views.py
@@ -42,8 +42,6 @@ from django.contrib.auth.decorators import login_required
from django.http import HttpResponseForbidden
from django.contrib import messages
-from beat.core.async import resolve_cpulimit_path
-
from ..experiments.models import Experiment
from .models import Environment, Worker, Queue
@@ -65,8 +63,6 @@ class Work:
def __setup__(self):
- Work.cpulimit = resolve_cpulimit_path(None)
- logger.debug("(path) cpulimit: `%s'", Work.cpulimit)
Work.process = utils.resolve_process_path()
logger.debug("(path) process: `%s'", Work.process)
Work.environments = utils.find_environments(None)
diff --git a/beat/web/code/api.py b/beat/web/code/api.py
old mode 100644
new mode 100755
index 7046ce6d46c355cdf8724079ad5c8836d07a69a4..b778eb7acf87d63c3ea7ba5f88fbfbeafcd523b6
--- a/beat/web/code/api.py
+++ b/beat/web/code/api.py
@@ -226,7 +226,7 @@ class RetrieveUpdateDestroyCodeView(RetrieveUpdateDestroyContributionView):
# - needed_dataformats
# - attestations
fields_to_remove = []
- if (request.user != db_object.author) and not(open_source):
+ if ((request.user != db_object.author) and not(open_source)) or db_object.is_binary():
fields_to_remove = ['code']
fields_to_return = self.get_serializer_fields(request, allow_sharing=(request.user == db_object.author),
diff --git a/beat/web/code/models.py b/beat/web/code/models.py
old mode 100644
new mode 100755
index b76b514ed27d0a653551e48c7939ea2eabe5712c..c7053c6bf88f832575fd079a479ee13f8a5521da
--- a/beat/web/code/models.py
+++ b/beat/web/code/models.py
@@ -55,8 +55,10 @@ class CodeManager(StoredContributionManager):
fork_of=None):
create = getattr(self, 'create_{}'.format(self.model.__name__.lower()))
- return create(author=author, name=name, short_description=short_description, description=description,
- declaration=declaration, code=code, version=version, previous_version=previous_version, fork_of=fork_of)
+ return create(author=author, name=name, short_description=short_description,
+ description=description, declaration=declaration, code=code,
+ version=version, previous_version=previous_version,
+ fork_of=fork_of)
def for_user(self, user, add_public=False):
if user.is_anonymous():
@@ -132,7 +134,7 @@ class CodeManager(StoredContributionManager):
# Figure out the language
language = declaration.get('language', 'unknown')
- language = getattr(Code, language.upper())
+ code_db.language = getattr(Code, language.upper())
# Check the provided source code
if code is None:
@@ -140,7 +142,7 @@ class CodeManager(StoredContributionManager):
code_db.source_code = previous_version.source_code
elif fork_of is not None:
code_db.source_code = fork_of.source_code
- else:
+ elif code_db.language != Code.CXX:
code_db.source_code = default.code
else:
code_db.source_code = code
@@ -194,19 +196,23 @@ class Code(StoredContribution):
# All possible values should be in sync with beat.core.utils
UNKNOWN = 'U'
- BINARY = 'B'
+ CXX = 'C'
MATLAB = 'M'
PYTHON = 'P'
R = 'R'
CODE_LANGUAGE = (
(UNKNOWN, 'Unknown'),
- (BINARY, 'Binary'),
+ (CXX, 'Cxx'),
(MATLAB, 'Matlab'),
(PYTHON, 'Python'),
(R, 'R'),
)
+ CODE_NAMES = {
+ CXX: 'C++',
+ }
+
#_____ Fields __________
@@ -470,6 +476,29 @@ class Code(StoredContribution):
return open_source
+ def is_binary(self):
+ return self.language in [Code.CXX]
+
+
+ def language_fullname(self):
+ if Code.CODE_NAMES.has_key(self.language):
+ return Code.CODE_NAMES[self.language]
+ return filter(lambda x: x[0] == self.language, Code.CODE_LANGUAGE)[0][1]
+
+
+ def json_language(self):
+ return Code.language_identifier(self.language)
+
+
+ @staticmethod
+ def language_identifier(db_language):
+ return filter(lambda x: x[0] == db_language, iter(Code.CODE_LANGUAGE))[0][1].lower()
+
+ @staticmethod
+ def language_db(language_identifier):
+ return filter(lambda x: x[1].lower() == language_identifier, iter(Code.CODE_LANGUAGE))[0][0]
+
+
#_____ Overrides __________
def save(self, *args, **kwargs):
@@ -477,10 +506,6 @@ class Code(StoredContribution):
# Invoke the base implementation
super(Code, self).save(*args, **kwargs)
- # If the filename has changed, move all the files
- if self.source_code_filename() != self.source_code_file.name:
- self._rename_file('source_code_file', self.source_code_filename())
-
# Ensures that the sharing informations are consistent
if self.sharing == Code.PUBLIC:
self.shared_with.clear()
@@ -514,7 +539,7 @@ class Code(StoredContribution):
return wrapper
- def _accessibility_for_user(self, user):
+ def _accessibility_for_user(self, user, without_usable=False):
"""Returns a tuple (<has_access>, <open_source>, <accessibility>), with <accessibility> being
either 'public', 'private', 'confidential' """
@@ -531,29 +556,29 @@ class Code(StoredContribution):
return (False, False, None)
elif self.sharing == Contribution.PUBLIC:
return (True, True, 'public')
- elif self.sharing == Contribution.USABLE:
+ elif not without_usable and (self.sharing == Contribution.USABLE):
return (True, False, 'confidential')
elif not user.is_anonymous():
if self.shared_with.filter(id=user.id).exists() or (self.shared_with_team.filter(members=user).count() > 0):
return (True, True, 'confidential')
- elif self.usable_by.filter(id=user.id).exists() or (self.usable_by_team.filter(members=user).count() > 0):
+ elif not without_usable and (self.usable_by.filter(id=user.id).exists() or (self.usable_by_team.filter(members=user).count() > 0)):
return (True, False, 'confidential')
return (False, False, None)
- def _accessibility_for_team(self, team):
+ def _accessibility_for_team(self, team, without_usable=False):
"""Team specific accessibility check """
if self.sharing == Contribution.PRIVATE:
return (False, False, None)
elif self.sharing == Contribution.PUBLIC:
return (True, True, 'public')
- elif self.sharing == Contribution.USABLE:
+ elif not without_usable and (self.sharing == Contribution.USABLE):
return (True, False, 'confidential')
elif self.shared_with_team.filter(id=team.id).exists():
return (True, True, 'confidential')
- elif self.usable_by_team.filter(id=team.id).exists():
+ elif not without_usable and self.usable_by_team.filter(id=team.id).exists():
return (True, False, 'confidential')
return (False, False, None)
diff --git a/beat/web/code/serializers.py b/beat/web/code/serializers.py
old mode 100644
new mode 100755
index 77f50e8dbd34ab9dee261b8e38510d376373e519..08307391e1f28323862f186ae95c288c6aafa560
--- a/beat/web/code/serializers.py
+++ b/beat/web/code/serializers.py
@@ -45,7 +45,7 @@ class CodeCreationSerializer(ContributionCreationSerializer):
code = serializers.CharField(required=False, allow_blank=True, trim_whitespace=False)
class Meta(ContributionCreationSerializer.Meta):
- fields = ContributionCreationSerializer.Meta.fields + ['code']
+ fields = ContributionCreationSerializer.Meta.fields + ['code', 'language']
#----------------------------------------------------------
@@ -65,11 +65,13 @@ class CodeSharingSerializer(SharingSerializer):
class CodeSerializer(ContributionSerializer):
opensource = serializers.SerializerMethodField()
+ language = serializers.SerializerMethodField()
modifiable = serializers.BooleanField()
+ valid = serializers.BooleanField()
class Meta(ContributionSerializer.Meta):
model = Code
- default_fields = ContributionSerializer.Meta.default_fields + ['opensource']
+ default_fields = ContributionSerializer.Meta.default_fields + ['opensource', 'language', 'valid']
extra_fields = ContributionSerializer.Meta.extra_fields + ['code']
exclude = ContributionSerializer.Meta.exclude + ['source_code_file']
@@ -85,6 +87,9 @@ class CodeSerializer(ContributionSerializer):
(has_access, open_source, accessibility) = obj.accessibility_for(user)
return open_source
+ def get_language(self, obj):
+ return Code.language_identifier(obj.language)
+
def get_accessibility(self, obj):
if obj.sharing == Code.PUBLIC:
return 'public'
diff --git a/beat/web/common/models.py b/beat/web/common/models.py
old mode 100644
new mode 100755
index 66e680a071844d6d34f3280d3cfd6e300e104701..50b3b68e6f10a69d65bd9f3eb8fa1b28bff81d65
--- a/beat/web/common/models.py
+++ b/beat/web/common/models.py
@@ -151,14 +151,14 @@ class Shareable(models.Model):
return (self.attestations.count() == 0)
return True
- def accessibility_for(self, user_or_team):
+ def accessibility_for(self, user_or_team, without_usable=False):
"""Returns a tuple (<has_access>, <accessibility>), with <accessibility> being
either 'public', 'private', 'confidential' """
if isinstance(user_or_team, User) or isinstance(user_or_team, AnonymousUser):
- return self._accessibility_for_user(user_or_team)
+ return self._accessibility_for_user(user_or_team, without_usable)
elif isinstance(user_or_team, Team):
- return self._accessibility_for_team(user_or_team)
+ return self._accessibility_for_team(user_or_team, without_usable)
else:
raise NotUserNorTeam
@@ -295,7 +295,7 @@ class Shareable(models.Model):
#_____ Protected Methods __________
- def _accessibility_for_user(self, user):
+ def _accessibility_for_user(self, user, without_usable=False):
"""User specific accessibility check """
if hasattr(self, 'author') and self.author == user:
@@ -317,7 +317,7 @@ class Shareable(models.Model):
return (False, None)
- def _accessibility_for_team(self, team):
+ def _accessibility_for_team(self, team, without_usable=False):
"""Team specific accessibility check """
if self.sharing == Shareable.PRIVATE:
@@ -726,11 +726,6 @@ class StoredContribution(Contribution):
# Invoke the base implementation
super(StoredContribution, self).save(*args, **kwargs)
- # If the filename has changed, move all the files
- if self.declaration_filename() != self.declaration_file.name:
- self._rename_file('declaration_file', self.declaration_filename())
- self._rename_file('description_file', self.description_filename())
-
#_____ Properties __________
diff --git a/beat/web/common/texts.py b/beat/web/common/texts.py
old mode 100644
new mode 100755
index 47a8a154abd2a73c6077d003e56541023d272d74..543fdaa59a2b5d1176913da3f6e939a88ee17017
--- a/beat/web/common/texts.py
+++ b/beat/web/common/texts.py
@@ -40,4 +40,5 @@ Messages = {
'format_name': 'The name for this dataformat (space-like characters will be automatically replaced by dashes)',
'name': 'The name for this object (space-like characters will be automatically replaced by dashes)',
'version': 'The version of this object (an integer starting from 1)',
+ 'shared_library': 'The compiled shared library file implementing your algorithm',
}
diff --git a/beat/web/experiments/static/experiments/js/panels.js b/beat/web/experiments/static/experiments/js/panels.js
index 9e0f1fd5b5a17d0f2194a78fa3b57ee4866da4b5..1a6751c1224690f9a5998a38c44dc4aa20812b08 100644
--- a/beat/web/experiments/static/experiments/js/panels.js
+++ b/beat/web/experiments/static/experiments/js/panels.js
@@ -48,6 +48,7 @@ beat.experiments.panels.Settings = function(panel_id, toolchain_name,
this.algorithms = null;
this.dataformats = null;
this.environments = null;
+ this.environments_all = null;
this.smart_datasets = null;
this.algorithm_mapping = algorithm_mapping;
this.url_prefix = url_prefix;
@@ -183,12 +184,22 @@ beat.experiments.panels.Settings.prototype.initialize = function(toolchain, conf
dataformats, datasets,
algorithms, environments)
{
- this.toolchain = toolchain;
- this.configuration = configuration;
- this.dataformats = dataformats;
- this.datasets = datasets;
- this.environments = environments;
- this.algorithms = algorithms;
+ this.toolchain = toolchain;
+ this.configuration = configuration;
+ this.dataformats = dataformats;
+ this.datasets = datasets;
+ this.environments_all = environments;
+ this.algorithms = algorithms;
+
+ // Separate the environments by languages
+ this.environments = {};
+
+ var languages = environments.languages();
+ for (var i = 0; i < languages.length; ++i)
+ {
+ var language = languages[i];
+ this.environments[language] = environments.filter(language);
+ }
// Update the environments
var components = [].concat(this.toolchain.blocks, this.toolchain.analyzers);
@@ -200,8 +211,8 @@ beat.experiments.panels.Settings.prototype.initialize = function(toolchain, conf
if ((configuration != null) && (configuration.environment != null))
{
- if (!this.environments.contains(configuration.environment.name, configuration.environment.version))
- this.configuration.setBlockEnvironment(component.name, this.environments.get(configuration.environment.name));
+ if (!this.environments_all.contains(configuration.environment.name, configuration.environment.version))
+ this.configuration.setBlockEnvironment(component.name, this.environments_all.get(configuration.environment.name));
}
}
@@ -912,6 +923,32 @@ beat.experiments.panels.Settings.prototype._onAlgorithmSelected = function(by_us
var algorithm_name = this.configuration.componentAlgorithm(this.current_block);
block_entry.algorithm = this.algorithms.get(algorithm_name);
+ // Update the environment if necessary
+ var block_env = this.configuration.blockEnvironment(this.current_block);
+ var default_env = this.configuration.defaultEnvironment();
+
+ if (!this.environments[block_entry.algorithm.language].contains(block_env.name, block_env.version))
+ {
+ if (!this.environments[block_entry.algorithm.language].contains(default_env.name, default_env.version))
+ {
+ var env = this.environments[block_entry.algorithm.language].getByIndex(0);
+
+ var config_env = {
+ name: env.name,
+ version: env.version,
+ };
+ this.configuration.setBlockEnvironment(this.current_block, config_env);
+ }
+ else
+ {
+ this.configuration.resetBlockEnvironmentAndQueue(this.current_block);
+ }
+ }
+ else if ((block_env.name == default_env.name) && (block_env.version == default_env.version))
+ {
+ this.configuration.resetBlockEnvironmentAndQueue(this.current_block);
+ }
+
// Update the selection controls
$(block_entry.algorithm_element).find('option').remove();
var option = $(document.createElement('option'));
@@ -1078,7 +1115,7 @@ beat.experiments.panels.Settings.prototype._displayParametersControls = function
this.configuration,
block_entry.block.name,
block_entry.algorithm,
- this.environments
+ this.environments[block_entry.algorithm.language]
);
$(block_entry.reset_button).show();
@@ -1453,6 +1490,7 @@ beat.experiments.panels.Parameters.prototype.initialize = function(configuration
var iterator = this.environments.iterator();
var selected_environment_index = null;
+ var first_python_environment_index = null;
var default_environment = this.configuration.defaultEnvironment();
if (default_environment !== null)
@@ -1468,31 +1506,39 @@ beat.experiments.panels.Parameters.prototype.initialize = function(configuration
var option = document.createElement('option');
option.textContent = environment.name + ' (' + environment.version + ')';
- option.value = selector_counter;
+ option.value = selector_counter;
- if ((default_environment != null) && (default_environment.name == environment.name) &&
- (default_environment.version == environment.version))
- {
- selected_environment_index = environment_selector.children.length;
- }
+ if ((default_environment != null) && (default_environment.name == environment.name) &&
+ (default_environment.version == environment.version))
+ {
+ selected_environment_index = environment_selector.children.length;
+ }
- environment_selector.appendChild(option);
- selector_counter += 1;
- }
+ if ((first_python_environment_index === null) && (environment.languages.indexOf('python') >= 0))
+ first_python_environment_index = environment_selector.children.length;
- if (selected_environment_index === null)
- selected_environment_index = 0;
+ environment_selector.appendChild(option);
+ selector_counter += 1;
+ }
- environment_selector.selectedIndex = selected_environment_index;
- this._onEnvironmentSelected();
+ if (selected_environment_index === null)
+ {
+ if (first_python_environment_index !== null)
+ selected_environment_index = first_python_environment_index;
+ else
+ selected_environment_index = 0;
+ }
+ environment_selector.selectedIndex = selected_environment_index;
+ this._onEnvironmentSelected();
- var selected_algorithms = this.configuration.algorithms();
- for (var i = 0; i < selected_algorithms.length; ++i)
- {
- var algorithm_name = selected_algorithms[i];
- this.addAlgorithm(selected_algorithms[i]);
- }
+
+ var selected_algorithms = this.configuration.algorithms();
+ for (var i = 0; i < selected_algorithms.length; ++i)
+ {
+ var algorithm_name = selected_algorithms[i];
+ this.addAlgorithm(selected_algorithms[i]);
+ }
}
diff --git a/beat/web/experiments/templates/experiments/panels/sharing.html b/beat/web/experiments/templates/experiments/panels/sharing.html
index 7a5a6a898fff8a56cc8cdcce012d4b7de2e42cf4..a9eca7ca407ccb9fba1842851862f3ab7b00dc01 100644
--- a/beat/web/experiments/templates/experiments/panels/sharing.html
+++ b/beat/web/experiments/templates/experiments/panels/sharing.html
@@ -109,14 +109,14 @@
<input id="public-radio" type="radio" name="sharing" value="public" checked="checked" onClick="$('#sharing-options').disable()"/>
Public
<p class="help">All users will be able to see, use and fork this
- dataformat</p>
+ experiment</p>
</label>
</div>
<div class="radio">
<label class="control-label">
<input id="shared-radio" type="radio" name="sharing" value="shared" onClick="$('#sharing-options').enable()"/> Shared
<p class="help">The users and teams indicated below will be
- able to see, use and fork this dataformat.</p>
+ able to see, use and fork this experiment.</p>
</label>
</div>
</div>
@@ -163,20 +163,24 @@
not use any data formats that belong to you.</p>
{% endif %}
- {% owner_algorithms object as algorithms %}
- {% if algorithms %}
+ {% owner_source_algorithms object as source_algorithms %}
+ {% owner_binary_algorithms object as binary_algorithms %}
+
+ {% if not source_algorithms and not binary_algorithms %}
+ <p class="help"><i class="fa fa-info-circle"></i> This experiment does
+ not use any algorithms that belong to you.</p>
+ {% endif %}
+
+ {% if source_algorithms %}
<p>The following algorithms and associated libraries will get the
same sharing permissions as well (i.e. <strong>readable</strong> and
executable by third-parties). Unselect those that will remain
closed-source:</p>
- {% else %}
- <p class="help"><i class="fa fa-info-circle"></i> This experiment does
- not use any algorithms that belong to you.</p>
{% endif %}
<div class="form-group">
- {% if algorithms %}
- {% for algo in algorithms %}
+ {% if source_algorithms %}
+ {% for algo in source_algorithms %}
<div id="visible-algorithms" class="checkbox{% if algo.get_sharing_display == 'Public' %} disabled{% endif %}" style="left-margin:1em">
<label>
<input type="checkbox" value="{{ algo.fullname }}" checked></input><a target="_blank" href="{{ algo.get_absolute_url }}">{{ algo.fullname }}</a> ({{ algo.get_sharing_display }})
@@ -185,6 +189,20 @@
{% endfor %}
{% endif %}
</div>
+
+ {% if binary_algorithms %}
+ <p>The following binary algorithms will get the
+ same sharing permissions as well (i.e. <strong>readable</strong> and
+ executable by third-parties).</p>
+ {% endif %}
+
+ <div class="form-group">
+ {% if binary_algorithms %}
+ {% for algo in binary_algorithms %}
+ <li><a target="_blank" href="{{ algo.get_absolute_url }}">{{ algo.fullname }}</a> ({{ algo.get_sharing_display }})</li>
+ {% endfor %}
+ {% endif %}
+ </div>
</div><!-- callout -->
</form>
</div><!-- edition -->
diff --git a/beat/web/experiments/templates/experiments/setup.html b/beat/web/experiments/templates/experiments/setup.html
index 61083eb1002a1d330938f9e7fb6b4ef3d9e27c9c..c3eee5666f1f406f6dbfbc42a3b4033266d7d060 100644
--- a/beat/web/experiments/templates/experiments/setup.html
+++ b/beat/web/experiments/templates/experiments/setup.html
@@ -402,9 +402,9 @@ jQuery(document).ready(function() {
{# get all algorithms #}
function getAlgorithms() {
var d = $.Deferred();
- var url = '{% url "api_algorithms:all" %}?fields=name,inputs,outputs,private,hash,parameters,short_description,splittable';
+ var url = '{% url "api_algorithms:all" %}?fields=name,inputs,outputs,private,hash,parameters,short_description,splittable,valid,language';
$.get(url).done(function(data) {
- var r = new beat.algorithms.models.AlgorithmsList(data);
+ var r = new beat.algorithms.models.AlgorithmsList(data, true);
advance_progressbar();
d.resolve(r);
}).fail(d.reject);
diff --git a/beat/web/experiments/templatetags/experiment_tags.py b/beat/web/experiments/templatetags/experiment_tags.py
old mode 100644
new mode 100755
index dd353a4125889d7d3d04010445e29a74e50c76ca..23444f35251c6a54c7aa894c93a9b1c19ad223c6
--- a/beat/web/experiments/templatetags/experiment_tags.py
+++ b/beat/web/experiments/templatetags/experiment_tags.py
@@ -277,8 +277,24 @@ def owner_algorithms(obj):
return obj.referenced_algorithms.filter(author=obj.author)
+@register.assignment_tag
+def owner_source_algorithms(obj):
+ '''Calculates the user algorithms in source code form for a given experiment'''
+
+ return [ x for x in obj.referenced_algorithms.filter(author=obj.author)
+ if not x.is_binary() ]
+
+
+@register.assignment_tag
+def owner_binary_algorithms(obj):
+ '''Calculates the user algorithms in binary form for a given experiment'''
+
+ return [ x for x in obj.referenced_algorithms.filter(author=obj.author)
+ if x.is_binary() ]
+
+
@register.assignment_tag
def owner_dataformats(obj):
- '''Calculates the user dataformats and algorithms for a given attestation'''
+ '''Calculates the user dataformats and algorithms for a given experiment'''
return [k for k in obj.all_needed_dataformats() if k.author == obj.author]
diff --git a/beat/web/libraries/migrations/0002_cxx_backend.py b/beat/web/libraries/migrations/0002_cxx_backend.py
new file mode 100644
index 0000000000000000000000000000000000000000..a0d8060e6a49bdb06c35fb3c7a0fc7e12053a420
--- /dev/null
+++ b/beat/web/libraries/migrations/0002_cxx_backend.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.11 on 2016-11-09 11:05
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('libraries', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='library',
+ name='language',
+ field=models.CharField(choices=[(b'U', b'Unknown'), (b'C', b'C++'), (b'M', b'Matlab'), (b'P', b'Python'), (b'R', b'R')], default=b'P', max_length=1),
+ ),
+ ]
diff --git a/beat/web/libraries/migrations/0003_auto_20161123_1218.py b/beat/web/libraries/migrations/0003_auto_20161123_1218.py
new file mode 100644
index 0000000000000000000000000000000000000000..a9cf143a86ab1482f4e710604bbf88f49ca1b858
--- /dev/null
+++ b/beat/web/libraries/migrations/0003_auto_20161123_1218.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.11 on 2016-11-23 12:18
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('libraries', '0002_cxx_backend'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='library',
+ name='language',
+ field=models.CharField(choices=[(b'U', b'Unknown'), (b'C', b'Cxx'), (b'M', b'Matlab'), (b'P', b'Python'), (b'R', b'R')], default=b'P', max_length=1),
+ ),
+ ]
diff --git a/beat/web/libraries/models.py b/beat/web/libraries/models.py
index 2c39ab852fa7d37b954bf1836721770685dd7182..e2ec84b87b38d8c8f59924dcf7ea941bd4120daa 100644
--- a/beat/web/libraries/models.py
+++ b/beat/web/libraries/models.py
@@ -202,6 +202,10 @@ class Library(Code):
return super(Library, self).deletable() and ((self.referencing.count() + self.used_by_algorithms.count()) == 0)
+ def valid(self):
+ return True # A library (at least for now) is always implemented in Python,
+ # thus always valid
+
def core(self):
return validate_library(self.declaration)
diff --git a/beat/web/plotters/migrations/0002_cxx_backend.py b/beat/web/plotters/migrations/0002_cxx_backend.py
new file mode 100644
index 0000000000000000000000000000000000000000..709a1a571bec83f4136767a5d0e97f93087ffcff
--- /dev/null
+++ b/beat/web/plotters/migrations/0002_cxx_backend.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.11 on 2016-11-09 11:05
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('plotters', '0001_initial'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='plotter',
+ name='language',
+ field=models.CharField(choices=[(b'U', b'Unknown'), (b'C', b'C++'), (b'M', b'Matlab'), (b'P', b'Python'), (b'R', b'R')], default=b'P', max_length=1),
+ ),
+ ]
diff --git a/beat/web/plotters/migrations/0003_auto_20161123_1218.py b/beat/web/plotters/migrations/0003_auto_20161123_1218.py
new file mode 100644
index 0000000000000000000000000000000000000000..7bae8a1cd36934689b86fe453cbe4c311038e46d
--- /dev/null
+++ b/beat/web/plotters/migrations/0003_auto_20161123_1218.py
@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.11 on 2016-11-23 12:18
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('plotters', '0002_cxx_backend'),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name='plotter',
+ name='language',
+ field=models.CharField(choices=[(b'U', b'Unknown'), (b'C', b'Cxx'), (b'M', b'Matlab'), (b'P', b'Python'), (b'R', b'R')], default=b'P', max_length=1),
+ ),
+ ]
diff --git a/beat/web/scripts/process.py b/beat/web/scripts/process.py
index 3c009f9522716502bdf57e36209816a745547eec..8c44eba51343210169f9120c8d67b49a25f1fdf9 100644
--- a/beat/web/scripts/process.py
+++ b/beat/web/scripts/process.py
@@ -30,14 +30,13 @@
Processes one split.
Usage:
- %(prog)s [--settings=<file>] [--cpulimit=<file>] [-v ...] <execute> <split>
+ %(prog)s [--settings=<file>] [-v ...] <split>
%(prog)s (-h | --help)
%(prog)s (-V | --version)
-Arguments:
+Arguments:
- <execute> The path to the base execution program for running the user code
<split> The primary-key of the split to treat by this subprocess
@@ -47,23 +46,18 @@ Options:
-v, --verbose Increases the output verbosity level
-S <file>, --settings=<file> The module name to the Django settings file
[default: beat.web.settings.settings]
- -C <file>, --cpulimit=<file> The path to the cpulimit program to use. If
- not set, CPU limiting is not enforced.
Examples:
To start the job split processing do the following:
- $ %(prog)s <path-to-execute> <split-id>
+ $ %(prog)s <split-id>
You can optionally pass the ``-v`` flag to start the worker with the logging
level set to ``INFO`` or ``-vv`` to set it to ``DEBUG``. By default, the
logging level is set to ``WARNING`` if no ``-v`` flag is passed.
- You can optionally also set the path to the ``cpulimit`` program to use. If
- it is not set, then CPU limiting will not be enforced.
-
"""
@@ -102,12 +96,7 @@ def main(user_input=None):
sys.exit(0)
def stop():
- import psutil
- for child in psutil.Process().children(recursive=True):
- if 'cpulimit' in child.name(): continue #only user processes
- child.kill()
- logger.info("Killing user process %d...", child.pid)
-
+ split.executor.kill()
message = "Force-stopped user processes for split `%s' for block " \
"`%s' of experiment `%s'" % \
(split, split.job.block.name,
@@ -126,7 +115,4 @@ def main(user_input=None):
signal.signal(signal.SIGTERM, handler)
signal.signal(signal.SIGINT, handler)
- split.process(
- execute=arguments['<execute>'],
- cpulimit=arguments['--cpulimit'],
- )
+ split.process()
diff --git a/beat/web/scripts/worker.py b/beat/web/scripts/worker.py
old mode 100644
new mode 100755
index c4a257ce3a27493011249caa961c7cdfd5f8f047..8b888b0f1fe9cf50b80d949955073065e20abb03
--- a/beat/web/scripts/worker.py
+++ b/beat/web/scripts/worker.py
@@ -31,7 +31,7 @@ Starts the worker process.
Usage:
%(prog)s [-v ... | --verbose ...] [--settings=<file>] [--period=<seconds>]
- [--cpulimit=<file>] [--environments=<path>] [--name=<name>]
+ [--environments=<path>] [--name=<name>]
%(prog)s (-h | --help)
%(prog)s (-V | --version)
@@ -42,10 +42,6 @@ Options:
-v, --verbose Increases the output verbosity level
-S <file>, --settings=<file> The module name to the Django settings
file [default: beat.web.settings.settings]
- -c <file>, --cpulimit=<file> The path to the cpulimit program to use. If
- not set, try to search in standard
- locations. If not found, CPU limiting is
- not enforced.
-e <path>, --environments=<path> The path to the installation root of
available environments.
-n <name>, --name=<name> The unique name of this worker on the
@@ -126,9 +122,6 @@ def main(user_input=None):
arguments['--name'])
# figure out paths to programs I need to use
- from beat.core.async import resolve_cpulimit_path
- cpulimit = resolve_cpulimit_path(arguments['--cpulimit'])
- logger.debug("(path) cpulimit: `%s'", cpulimit)
process = utils.resolve_process_path()
logger.debug("(path) process: `%s'", process)
@@ -162,7 +155,7 @@ def main(user_input=None):
start = time.time()
logger.debug("Starting work cycle...")
- worker.work(environments, cpulimit, process)
+ worker.work(environments, process)
duration = time.time() - start
if duration < timing:
time.sleep(timing - duration)
diff --git a/beat/web/settings/test.py b/beat/web/settings/test.py
index 9bb7d4dc121920241d630c0f8c4430553968aa88..4d45a26d9a48141bf8e2b1b6ae0d3f95ce8662a3 100644
--- a/beat/web/settings/test.py
+++ b/beat/web/settings/test.py
@@ -29,6 +29,8 @@
from .settings import *
+TEST_CONFIGURATION = True
+
DEBUG = False
TEMPLATES[0]['OPTIONS']['debug'] = DEBUG
@@ -46,6 +48,7 @@ if 'beat.cmdline' in sys.argv:
LOGGING['handlers']['console']['level'] = 'DEBUG'
LOGGING['loggers']['beat.core']['handlers'] = ['discard']
+LOGGING['loggers']['beat.web.utils.management.commands']['handlers'] = ['discard']
PREFIX = os.environ.get('BEAT_TEST_PREFIX', os.path.realpath('./test_prefix'))
ALGORITHMS_ROOT = os.path.join(PREFIX, 'algorithms')
diff --git a/beat/web/utils/management/commands/install.py b/beat/web/utils/management/commands/install.py
old mode 100644
new mode 100755
index 8e8cba72e46b321ab8f4df1a22a98c00fe325585..e5f753cfa1a5e009de3c7002cb5b3c7532cdfc9e
--- a/beat/web/utils/management/commands/install.py
+++ b/beat/web/utils/management/commands/install.py
@@ -1043,9 +1043,10 @@ class Command(BaseCommand):
# Sets up the queue and environments
setup_environment(arguments['queue_configuration'],
arguments['verbosity'])
- from ....backend.models import Environment, Queue
- environment = Environment.objects.first()
- queue = Queue.objects.first()
+ from ....backend.models import Environment, EnvironmentLanguage, Queue
+ from ....code.models import Code
+ environment = EnvironmentLanguage.objects.filter(language=Code.PYTHON).first().environment
+ queue = environment.queues.first()
# Iterates over projects to install
for project in ['system'] + arguments['project']:
diff --git a/buildout.cfg b/buildout.cfg
index 3a20a2e1bafd2f66dbc79a51f72d0eeb709943fa..e1f99b7d2a4936e37f4d466ad33fae0135e94629 100644
--- a/buildout.cfg
+++ b/buildout.cfg
@@ -15,10 +15,10 @@ develop = .
versions = versions
[versions]
-django = >=1.8
-django-rest-swagger = >=0.3.2
+django = >=1.9,<1.10
+django-rest-swagger = >=0.3.2,<0.3.3
django-guardian = >=1.3
-djangorestframework = >=3.2
+djangorestframework = >=3.2,<3.3
django-activity-stream = >= 0.6.0
[sysegg]
@@ -89,7 +89,6 @@ beat.core = git git@gitlab.idiap.ch:beat/beat.core
beat.cmdline = git git@gitlab.idiap.ch:beat/beat.cmdline
beat.backend.python = git git@gitlab.idiap.ch:beat/beat.backend.python
beat.examples = git git@gitlab.idiap.ch:beat/beat.examples egg=false
-cpulimit = git https://github.com/opsengine/cpulimit rev=v0.2 egg=false
[scripts]
recipe = bob.buildout:scripts
@@ -112,6 +111,7 @@ packages = jquery#~1.11.3
jquery-dateFormat#~1.0.2
jquery-ui#~1.10.4
jquery.cookie#~1.4.1
+ jquery-file-upload#~9.14.0
fontawesome#~4.5.0
codemirror#~5.10.0
bootstrap#~3.3.6
@@ -121,7 +121,7 @@ packages = jquery#~1.11.3
mousetrap#~1.5.3
raphael#~2.1.4
spectrum#~1.7.1
- git@github.com:joshaven/string_score#~0.1.22
+ https://github.com/joshaven/string_score.git#~0.1.22
chosen-bootstrap#~1.1.0
angularjs#~1.4.5
angular-ui-router#~0.2.15
diff --git a/doc/user/algorithms/guide.rst b/doc/user/algorithms/guide.rst
index defd1cc07a1427be5480eec8dc5eb9bdf0be1444..e2b2aa82d8ec2f368be452abf9036397626a4658 100644
--- a/doc/user/algorithms/guide.rst
+++ b/doc/user/algorithms/guide.rst
@@ -51,14 +51,15 @@ toolchain. Flow synchronization is determined by data units produced from a
dataset and injected into the toolchain.
Code for algorithms may be implemented in any programming language supported by
-|project|. At present, only a Python backend has been integrated and,
-therefore, algorithms are expected to be implemented in this language. (In
-future, other backends will be added to |project|.) Python code implementing a
-certain algorithm can be created using our web-based :ref:`algorithm editor`.
+|project|. At present, only two backends have been integrated, supporting Python
+and C++, therefore, algorithms are expected to be implemented in one of those
+languages. (In future, other backends will be added to |project|.) Python code implementing a certain algorithm can be created using our web-based
+:ref:`algorithm editor`. C++ based algorithms must be compiled using a provided
+docker container, and uploaded on the platform (see :ref:`binary algorithms`).
|project| treats algorithms as objects that are derived from the class
-``Algorithm``. To define a new algorithm, at least one method must be
-implemented:
+``Algorithm`` (in Python) or ``IAlgorithm`` (in C++). To define a new algorithm,
+at least one method must be implemented:
* ``process()``: the method that actually processes input and produces
outputs.
@@ -71,10 +72,25 @@ Python):
class Algorithm:
- def process(self, inputs, outputs):
+ def process(self, inputs, outputs):
# here, you read inputs, process and write results to outputs
+Here is the equivalent example in C++:
+
+.. code-block:: c++
+ :linenos:
+
+ class Algorithm: public IAlgorithm
+ {
+ public:
+ virtual bool process(const InputList& inputs, const OutputList& outputs)
+ {
+ // here, you read inputs, process and write results to outputs
+ }
+ };
+
+
One particularity of the |project| platform is how the data-flow through a
given toolchain is synchronized. The platform is responsible for extracting
data units (images, speech-segments, videos, etc.) from the database and
@@ -112,26 +128,63 @@ An example code showing how to implement an algorithm in this configuration is s
.. code-block:: python
:linenos:
- class Algorithm:
+ class Algorithm:
- def process(self, inputs, outputs):
+ def process(self, inputs, outputs):
- # to read the field "value" on the "in" input, use "data"
- # a possible declaration of "user/format/1" would be:
- # {
- # "value": ...
- # }
- value = inputs['in'].data.value
+ # to read the field "value" on the "in" input, use "data"
+ # a possible declaration of "user/format/1" would be:
+ # {
+ # "value": ...
+ # }
+ value = inputs['in'].data.value
- # do your processing and create the "output" value
- output = magical_processing(value)
+ # do your processing and create the "output" value
+ output = magical_processing(value)
+
+ # to write "output" into the relevant endpoint use "write"
+ # a possible declaration of "user/other/1" would be:
+ # {
+ # "value": ...
+ # }
+ outputs['out'].write({'value': output})
+
+ # No error occurred, so return True
+ return True
+
+
+.. code-block:: c++
+ :linenos:
- # to write "output" into the relevant endpoint use "write"
- # a possible declaration of "user/other/1" would be:
- # {
- # "value": ...
- # }
- outputs['out'].write({'value': output})
+ class Algorithm: public IAlgorithm
+ {
+ public:
+ virtual bool process(const InputList& inputs, const OutputList& outputs)
+ {
+ // to read the field "value" on the "in" input, use "data"
+ // a possible declaration of "user/format/1" would be:
+ // {
+ // "value": ...
+ // }
+ auto value = inputs["in"]->data<user::format_1>()->value;
+
+ // do your processing and create the "output" value
+ auto output = magical_processing(value);
+
+ // to write "output" into the relevant endpoint use "write"
+ // a possible declaration of "user/other/1" would be:
+ // {
+ // "value": ...
+ // }
+ auto result = new user::other_1();
+ result->value = output;
+
+ outputs["out"]->write(result);
+
+ # No error occurred, so return true
+ return true;
+ }
+ };
In this example, the platform will call the user algorithm every time a new
@@ -159,18 +212,41 @@ shown below:
.. code-block:: python
:linenos:
- class Algorithm:
+ class Algorithm:
+
+ def process(self, inputs, outputs):
+
+ i1 = inputs['in'].data.value
+ i2 = inputs['in2'].data.value
+
+ out = magical_processing(i1, i2)
+
+ outputs['out'].write({'value': out})
- def process(self, inputs, outputs):
+ return True
- i1 = inputs['in'].data.value
- i2 = inputs['in2'].data.value
- out = magical_processing(i1, i2)
+.. code-block:: c++
+ :linenos:
+
+ class Algorithm: public IAlgorithm
+ {
+ public:
+ virtual bool process(const InputList& inputs, const OutputList& outputs)
+ {
+ auto i1 = inputs["in"]->data<user::format_1>()->value;
+ auto i2 = inputs["in2"]->data<user::format_1>()->value;
+
+ auto out = magical_processing(i1, i2);
- outputs['out'].write({'value': out})
+ auto result = new user::other_1();
+ result->value = out;
- return True
+ outputs["out"]->write(result);
+
+ return true;
+ }
+ };
You should notice that we still don't require any sort of ``for`` loops! The
@@ -201,20 +277,50 @@ The example below illustrates how such an algorithm could be implemented:
.. code-block:: python
:linenos:
- class Algorithm:
+ class Algorithm:
- def __init__(self):
- self.objs = []
+ def __init__(self):
+ self.objs = []
- def process(self, inputs, outputs):
- self.objs.append(inputs['in'].data.value) #accumulates
+ def process(self, inputs, outputs):
+ self.objs.append(inputs['in'].data.value) # accumulates
if inputs['in2'].isDataUnitDone():
out = magical_processing(self.objs)
outputs['out'].write({'value': out})
self.objs = [] #reset accumulator for next label
- return True
+ return True
+
+
+.. code-block:: c++
+ :linenos:
+
+ class Algorithm: public IAlgorithm
+ {
+ public:
+ virtual bool process(const InputList& inputs, const OutputList& outputs)
+ {
+ objs.push_back(inputs["in"]->data<user::format_1>()->value); // accumulates
+
+ if (inputs["in2"]->isDataUnitDone())
+ {
+ auto out = magical_processing(objs);
+
+ auto result = new user::other_1();
+ result->value = out;
+
+ outputs["out"]->write(result);
+
+ objs.clear(); // reset accumulator for next label
+ }
+
+ return true;
+ }
+
+ public:
+ std::vector<float> objs;
+ };
Here, the units received at the endpoint ``in`` are accumulated as long as the
@@ -247,34 +353,71 @@ unsynchronized input (``in3``).
.. code-block:: python
:linenos:
- class Algorithm:
+ class Algorithm:
+
+ def __init__(self):
+ self.models = None
+
+ def process(self, inputs, outputs):
+ # N.B.: this will be called for every unit in `in'
+
+ # Loads the "model" data at the beginning, once
+ if self.models is None:
+ self.models = []
+ while inputs['in3'].hasMoreData():
+ inputs['in3'].next()
+ self.models.append(inputs['in3'].data.value)
+
+ # Processes the current input in `in' and `in2', apply the
+ # model/models
+ out = magical_processing(inputs['in'].data.value,
+ inputs['in2'].data.value,
+ self.models)
+ # Writes the output
+ outputs.write({'value': out})
- def __init__(self):
+ return True
- self.models = None
+.. code-block:: c++
+ :linenos:
+
+ class Algorithm: public IAlgorithm
+ {
+ public:
+ virtual bool process(const InputList& inputs, const OutputList& outputs)
+ {
+ // N.B.: this will be called for every unit in `in'
+
+ // Loads the "model" data at the beginning, once
+ if (models.empty())
+ {
+ while (inputs["in3"]->hasMoreData())
+ {
+ inputs["in3"]->next();
+ auto model = inputs["in3"]->data<user::model_1>();
+ models.push_back(*model);
+ }
+ }
- def process(self, inputs, outputs):
- # N.B.: this will be called for every unit in `in'
+ // Processes the current input in `in' and `in2', apply the model/models
+ auto out = magical_processing(inputs["in"]->data<user::format_1>()->value,
+ inputs["in2"]->data<user::format_1>()->value,
+ models);
- # Loads the "model" data at the beginning, once
- if self.models is None:
- self.models = []
- while inputs['in3'].hasMoreData():
- inputs['in3'].next()
- self.models.append(inputs['in3'].data.value)
+ // Writes the output
+ auto result = new user::other_1();
+ result->value = out;
- # Processes the current input in `in' and `in2', apply the
- # model/models
- out = magical_processing(inputs['in'].data.value,
- inputs['in2'].data.value,
- self.models)
+ outputs["out"]->write(result);
- # Writes the output
- outputs.write({'value': out})
+ return true;
+ }
- return True
+ public:
+ std::vector<user::model_1> models;
+ };
It may happen that you have several inputs which are synchronized together, but
@@ -284,34 +427,73 @@ it is safer to treat inputs using their *group*. For example:
.. code-block:: python
:linenos:
- class Algorithm:
+ class Algorithm:
+ def __init__(self):
+ self.models = None
- def __init__(self):
- self.models = None
+ def process(self, inputs, outputs):
+ # N.B.: this will be called for every unit in `in'
+ # Loads the "model" data at the beginning, once
+ if self.models is None:
+ self.models = []
+ group = inputs.groupOf('in3')
+ while group.hasMoreData():
+ group.next() #synchronously advances the data
+ self.models.append(group['in3'].data.value)
- def process(self, inputs, outputs):
- # N.B.: this will be called for every unit in `in':
+ # Processes the current input in `in' and `in2', apply the model/models
+ out = magical_processing(inputs['in'].data.value,
+ inputs['in2'].data.value,
+ self.models)
- # Loads the "model" data at the beginning, once
- if self.models is None:
- self.models = []
- group = inputs.groupOf('in3')
- while group.hasMoreData():
- group.next() #synchronously advances the data
- self.models.append(group['in3'].data.value)
+ # Writes the output
+ outputs.write({'value': out})
- # Processes the current input in `in' and `in2', apply the model/models
- out = magical_processing(inputs['in'].data.value,
- inputs['in2'].data.value,
- self.models)
+ return True
- # Writes the output
- outputs.write({'value': out})
- return True
+.. code-block:: c++
+ :linenos:
+
+ class Algorithm: public IAlgorithm
+ {
+ public:
+ virtual bool process(const InputList& inputs, const OutputList& outputs)
+ {
+ // N.B.: this will be called for every unit in `in'
+
+ // Loads the "model" data at the beginning, once
+ if (models.empty())
+ {
+ auto group = inputs->groupOf("in3");
+ while (group->hasMoreData())
+ {
+ group->next(); // synchronously advances the data
+ auto model = group["in3"]->data<user::model_1>();
+ models.push_back(*model);
+ }
+ }
+
+ // Processes the current input in `in' and `in2', apply the model/models
+ auto out = magical_processing(inputs["in"]->data<user::format_1>()->value,
+ inputs["in2"]->data<user::format_1>()->value,
+ models);
+
+ // Writes the output
+ auto result = new user::other_1();
+ result->value = out;
+
+ outputs["out"]->write(result);
+
+ return true;
+ }
+
+ public:
+ std::vector<user::model_1> models;
+ };
In practice, encoding your algorithms using *groups* instead of looping over
@@ -359,7 +541,7 @@ image below.
There are two types of algorithm in the editor: Analyzer, and Splittable.
Analyzer algorithms are special algorithms where the purpose is to generate
statistics about the processing results (graphs, means, variances, etc.).
-Usual, biometric data processing algorithms are of type Splittable, indicating
+Usually, biometric data processing algorithms are of type Splittable, indicating
to the platform that these algorithms can be executed in a distributed fashion,
depending on the available computing resources.
@@ -378,7 +560,7 @@ You should see a web-page similar to what is displayed below:
.. image:: img/algorithm_new.*
-For instructions on how to create an algorithm from scratch, please refer to the Section of `algorithm editor`_.??????
+For instructions on how to create an algorithm from scratch, please refer to the Section of `algorithm editor`_.
Edit an existing algorithm
@@ -409,15 +591,20 @@ Please refer to the Section of `algorithm editor`_ for creating an algorithm.
Editor
------
-To create an algorithm, there are six sections which are:
+To create an algorithm, there are seven sections which are:
* Name: the name of algorithm.
* Algorithm type: Analyzer or Splittable.
+ * Language: The language used to implement the algorithm (Python or C++).
* Documentation: This is used to describe your algorithm.
- * Source code: The (Python) code implementing the algorithm.
* Inputs / Outputs: Define the properties of the Input and Output endpoints for this algorithm.
* Parameters: Define the configuration-parameters for the algorithm.
+
+For Python-based algorithms only:
+
* Libraries: If there are functions in a library, you can add them for the algorithm to use.
+ * Source code: The (Python) code implementing the algorithm.
+
You should see a webpage similar to what is displayed below:
@@ -446,6 +633,89 @@ your algorithm code, to help with your debugging.
very last 4 kilobytes of these streams is kept for user inspection.
+.. _binary algorithms:
+
+Implementing an algorithm in C++
+--------------------------------
+
+Prerequisite: Configure your command-line client
+================================================
+
+In order to ensure that your compiled algorithm will works on the |project| platform,
+you must compile it using our docker image called *beats/client*. Once downloaded,
+you'll need to configure the command-line tool to access your account on the |project|
+platform:
+
+.. code-block:: bash
+
+ $ docker run -ti beats/client:0.1.5 bash
+ /# cd home
+ /home# beat config set user <your_user_name>
+ /home# beat config set token "<your_token>"
+ /home# beat config save
+
+Here, ``<your_user_name>`` is your username on the |project| platform, and
+``<your_token>`` can be retrieved from your settings page. Note that we use the
+``/home`` folder to save everything, but feel free to use the one you want.
+
+
+Algorithm compilation
+=====================
+
+To implement an algorithm in C++, follow the following steps:
+
+ 1. Create the algorithm on the |project| platform, by selecting the C++ language.
+ Declare all the needed inputs, outputs and parameters.
+
+ 2. Using the ``beat`` command-line tool, download the algorithm declaration from the
+ |project| platform (note that all the necessary data formats will be dowloaded too):
+
+.. code-block:: bash
+
+ /home# beat algorithms pull <your_user_name>/<algorithm_name>/<version>
+
+At this point, the folder ``/home/algorithms/<your_user_name>/<algorithm_name>/``
+will contain the declaration of your algorithm in JSON format, and ``/home/dataformats/``
+will contain the declaration files of the data formats used by the algorithm.
+
+ 3. Generate the C++ files corresponding to the algorithm declaration:
+
+.. code-block:: bash
+
+ /home# generate_cxx.py . <your_user_name>/<algorithm_name>/<version>
+
+At this point, the folder ``/home/algorithms/<your_user_name>/<algorithm_name>/``
+will contain a few new C++ files:
+
+ * one header/source file for each needed data format
+ * ``beat_setup.h`` and ``beat_setup.cpp``: used by the platform to learn everything
+ it needs to know about your algorithm
+ * ``algorithm.h`` and ``algorithm.cpp``: you will implement your algorithm in those
+ files
+
+Feel free to add as many other files as you need for your implementation.
+
+ 4. Implement your algorithm in ``algorithm.h`` and ``algorithm.cpp``
+
+ 5. Compile your code as a shared library (an example CMake file was generated, you can
+ either modify it to add your own files or use another build system if you want). Note
+ that the |project| platform expect you to upload one and only one *shared library*, so
+ if your algorithm has any dependencies, you must link them statically inside the shared
+ library:
+
+.. code-block:: bash
+
+ /home# cd algorithms/<your_user_name>/<algorithm_name>/
+ /home/algorithms/<your_user_name>/<algorithm_name># mkdir build
+ /home/algorithms/<your_user_name>/<algorithm_name># cd build
+ /home/algorithms/<your_user_name>/<algorithm_name>/build# cmake ..
+ /home/algorithms/<your_user_name>/<algorithm_name>/build# make
+
+This will produce a file called ``<version>.so`` in the ``/home/algorithms/<your_user_name>/<algorithm_name>/`` folder.
+
+ 6. Upload the shared library on the |project| platform, from the algorithm page.
+
+
Sharing an Algorithm
---------------------