Merge branch 'v2' into 'master'

V2

Closes #244, #226, #242, #232, #181, #234, #233, #231, #223, #222, #221, #215, #214, #183, #212, #178, #204, #191, #189, and #190

See merge request !112

.flake8

+[flake8]
+max-line-length = 80
+select = B,C,E,F,W,T4,B9,B950
+ignore = E501, W503, E203

.gitignore

-
-# Created by https://www.gitignore.io/api/node
-
-### Node ###
-# Logs
-logs
-*.log
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-
-# Runtime data
-pids
-*.pid
-*.seed
-*.pid.lock
-
-# Directory for instrumented libs generated by jscoverage/JSCover
-lib-cov
-
-# Coverage directory used by tools like istanbul
-coverage
-
-# nyc test coverage
-.nyc_output
-
-# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
-.grunt
-
-# Bower dependency directory (https://bower.io/)
-bower_components
-
-# node-waf configuration
-.lock-wscript
-
-# Compiled binary addons (http://nodejs.org/api/addons.html)
-build/Release
-
-# Dependency directories
-node_modules/
-jspm_packages/
-
-# Typescript v1 declaration files
-typings/
-
-# Optional npm cache directory
-.npm
-
-# Optional eslint cache
-.eslintcache
-
-# Optional REPL history
-.node_repl_history
-
-# Output of 'npm pack'
-*.tgz
-
-# Yarn Integrity file
-.yarn-integrity
-
-# dotenv environment variables file
-.env
-
-
-# End of https://www.gitignore.io/api/node
-
-# built files
-dist/
-# docs cache/build
-styleguide/
-
-flow-typed/
-
-coverage/
-
-# user conf file for app
-user_conf.json
-
 # pycache
 __pycache__
 
@@ -109,8 +31,8 @@ miniconda.cached/
 conda/recipe_append.yaml
 conda-bld/
 
-# built JS files
-beat/editor/js
+# users prefix
 prefix/
-conda/js/package.json
-conda/js/webpack.config.js
+
+# development buildout
+src/

.gitlab-ci.yml

 include: 'https://gitlab.idiap.ch/bob/bob.devtools/raw/master/bob/devtools/data/gitlab-ci/single-package.yaml'
-
-# Redefines the pipeline order for this package only
-stages:
-  - build
-  - browser-tests
-  - deploy
-  - pypi
-
-# Docker host based testing (must be run inside dind or docker-enabled host)
-browser_test:
-  extends: .test_linux_template
-  stage: browser-tests
-  image: docker.idiap.ch/beat/ci.env.editor:0.0.1r0
-  variables:
-    BEAT_BROWSER_TESTS: "true"
-    LANG: "en_US.UTF-8"
-    LANGUAGE: "en_US:en"
-    LC_ALL: "en_US.UTF-8"
-  dependencies:
-    - build_linux_36
-  before_script:
-    - curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
-    - python3 bootstrap.py -vv channel base
-    - source ${CONDA_ROOT}/etc/profile.d/conda.sh
-    - conda activate base
-    - apt-get update > /dev/null
-    - apt-get install -y locales > /dev/null
-    - echo "en_US UTF-8" > /etc/locale.gen
-    - locale-gen en_US.UTF-8
-    - rm -rf ${CI_PROJECT_DIR}/sphinx

.gitlab/issue_templates/suggestion.md

@@ -17,4 +17,4 @@ Relevant links / references
 
 (Paste any relevant links / image for the implementation of the suggestion)
 
-/label ~suggestion
\ No newline at end of file
+/label ~suggestion

.pre-commit-config.yaml

+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+  - repo: https://github.com/ambv/black
+    rev: stable
+    hooks:
+    - id: black
+      language_version: python3.6
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: debug-statements
+      - id: check-added-large-files
+      - id: check-docstring-first
+      - id: flake8
+      - id: check-yaml
+        exclude: conda/meta.yaml
+  - repo: https://github.com/PyCQA/bandit
+    rev: 'master' # Update me!
+    hooks:
+      - id: bandit
+        exclude: beat/editor/test
+  - repo: local
+    hooks:
+      - id: sphinx-build
+        name: sphinx build
+        entry: python -m sphinx.cmd.build
+        args: [-a, -E, -W, doc, sphinx]
+        language: system
+        files: ^doc/
+        types: [file]
+        pass_filenames: false
+      - id: sphinx-doctest
+        name: sphinx doctest
+        entry: python -m sphinx.cmd.build
+        args: [-a, -E, -b, doctest, doc, sphinx]
+        language: system
+        files: ^doc/
+        types: [file]
+        pass_filenames: false

MANIFEST.in

 include LICENSE.AGPL README.rst version.txt requirements.txt buildout.cfg
-recursive-include doc conf.py *.rst *.png *.ico
+recursive-include doc conf.py *.rst *.png *.ico *.md
 recursive-include beat/editor/templates *.jinja2
-recursive-include beat/editor/js *

beat/__init__.py

@@ -27,4 +27,5 @@
 
 # see https://docs.python.org/3/library/pkgutil.html
 from pkgutil import extend_path
+
 __path__ = extend_path(__path__, __name__)

beat/editor/__init__.py

+# vim: set fileencoding=utf-8 :
+###############################################################################
+#                                                                             #
+# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/           #
+# Contact: beat.support@idiap.ch                                              #
+#                                                                             #
+# This file is part of the beat.editor module of the BEAT platform.           #
+#                                                                             #
+# Commercial License Usage                                                    #
+# Licensees holding valid commercial BEAT licenses may use this file in       #
+# accordance with the terms contained in a written agreement between you      #
+# and Idiap. For further information contact tto@idiap.ch                     #
+#                                                                             #
+# Alternatively, this file may be used under the terms of the GNU Affero      #
+# Public License version 3 as published by the Free Software and appearing    #
+# in the file LICENSE.AGPL included in the packaging of this file.            #
+# The BEAT platform is distributed in the hope that it will be useful, but    #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
+#                                                                             #
+# You should have received a copy of the GNU Affero Public License along      #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
+#                                                                             #
+###############################################################################

beat/editor/test/test_resources.py → beat/editor/backend/__init__.py

@@ -24,39 +24,3 @@
 # with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
 #                                                                             #
 ###############################################################################
-
-# test the resources.py file
-# (mostly endpoints and working with the filesystem)
-
-import nose.tools
-import os
-
-from .. import resources
-
-# the func names the endpoint the given name
-def test_check_valid_generated_endpoint_name():
-    name = 'TestEndpoint'
-    endpoint = resources.gen_endpoint(name)
-    nose.tools.eq_(endpoint.__name__, name)
-
-# the func doesnt accept non-entity names
-@nose.tools.raises(AssertionError)
-def test_assert_valid_entity_invalid():
-    resources.assert_valid_entity('notanentity')
-
-# the func parses this file
-def test_path_to_dict_file():
-    currfile = os.path.realpath(__file__)
-    res = resources.path_to_dict(currfile)
-    # in python 3 the first case works but in python 2 the second case works
-    assert res == {'name': 'test_resources.py', 'type': 'file'} or res == {'name': 'test_resources.pyc', 'type': 'file'}
-
-# the func parses this folder
-def test_path_to_dict_folder():
-    currfolder = os.path.dirname(os.path.realpath(__file__))
-    res = resources.path_to_dict(currfolder)
-    nose.tools.eq_(res['name'], 'test')
-    nose.tools.eq_(res['type'], 'directory')
-    nose.tools.ok_({'name': '__init__.py', 'type': 'file'} in res['children'])
-    nose.tools.ok_({'name': 'test_resources.py', 'type': 'file'} in res['children'])
-

beat/editor/backend/asset.py

+# vim: set fileencoding=utf-8 :
+###############################################################################
+#                                                                             #
+# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/           #
+# Contact: beat.support@idiap.ch                                              #
+#                                                                             #
+# This file is part of the beat.editor module of the BEAT platform.           #
+#                                                                             #
+# Commercial License Usage                                                    #
+# Licensees holding valid commercial BEAT licenses may use this file in       #
+# accordance with the terms contained in a written agreement between you      #
+# and Idiap. For further information contact tto@idiap.ch                     #
+#                                                                             #
+# Alternatively, this file may be used under the terms of the GNU Affero      #
+# Public License version 3 as published by the Free Software and appearing    #
+# in the file LICENSE.AGPL included in the packaging of this file.            #
+# The BEAT platform is distributed in the hope that it will be useful, but    #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
+#                                                                             #
+# You should have received a copy of the GNU Affero Public License along      #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
+#                                                                             #
+###############################################################################
+
+import importlib
+import simplejson as json
+
+import beat.core
+
+from enum import Enum, unique
+
+from beat.backend.python import utils
+from beat.core.schema import validate
+from beat.cmdline import common
+
+
+@unique
+class AssetType(Enum):
+    """All possible assets available on the BEAT platform"""
+
+    UNKNOWN = ("unknown", None)
+    ALGORITHM = ("algorithms", beat.core.algorithm.Algorithm)
+    DATABASE = ("databases", beat.core.database.Database)
+    DATAFORMAT = ("dataformats", beat.core.dataformat.DataFormat)
+    EXPERIMENT = ("experiments", beat.core.experiment.Experiment)
+    LIBRARY = ("libraries", beat.core.library.Library)
+    PLOTTER = ("plotters", beat.core.plotter.Plotter)
+    PLOTTERPARAMETER = (
+        "plotterparameters",
+        beat.core.plotterparameter.Plotterparameter,
+    )
+    PROTOCOLTEMPLATE = (
+        "protocoltemplates",
+        beat.core.protocoltemplate.ProtocolTemplate,
+    )
+    TOOLCHAIN = ("toolchains", beat.core.toolchain.Toolchain)
+
+    def __init__(self, path, klass):
+        self.path = path
+        self.klass = klass
+
+    @property
+    def storage(self):
+        mod = importlib.import_module(self.klass.__module__)
+        return getattr(mod, "Storage")
+
+    @staticmethod
+    def from_path(path):
+        for asset_type in AssetType:
+            if asset_type.path == path:
+                return asset_type
+        raise RuntimeError("Unknown asset path {}".format(path))
+
+    def can_create(self):
+        """Returns whether a new asset can be created from scratch"""
+
+        return self not in [self.UNKNOWN, self.EXPERIMENT]
+
+    def has_versions(self):
+        """Returns whether a new version of this asset can be created"""
+
+        return self not in [self.UNKNOWN, self.EXPERIMENT]
+
+    def can_fork(self):
+        """Returns whether a new asset can be forked"""
+
+        return self not in [self.UNKNOWN, self.DATABASE, self.PROTOCOLTEMPLATE]
+
+    def split_count(self):
+        """Returns the number of "/" that should be part of its name"""
+
+        if self == self.UNKNOWN:
+            return 0
+        elif self == self.EXPERIMENT:
+            return 5
+        elif self not in [self.DATABASE, self.PROTOCOLTEMPLATE]:
+            return 2
+        else:
+            return 1
+
+    def validate(self, data):
+        """Runs the schema validation and returns whether an asset is valid
+
+        :param data str: asset content
+        """
+
+        if self == self.UNKNOWN:
+            raise RuntimeError("Trying to validate unknown type")
+        return validate(self.name.lower(), data)
+
+    def create_new(self, prefix, name):
+        """Create a new asset from a prototype
+
+        :param prefix str: Path to the prefix
+        :param name str: name of the asset
+        """
+
+        if self == self.UNKNOWN:
+            raise RuntimeError("Trying to create an asset of unknown type")
+        success = common.create(prefix, self.name.lower(), [name])
+        return success == 0
+
+    def create_new_version(self, prefix, name):
+        """Create a new version of the asset
+
+        :param prefix str: Path to the prefix
+        :param name str: name of the asset
+        """
+
+        if self == self.UNKNOWN:
+            raise RuntimeError(
+                "Trying to create a new version of an asset of unknown type"
+            )
+        success = common.new_version(prefix, self.name.lower(), name)
+        return success == 0
+
+    def fork(self, prefix, source, destination):
+        """Fork an asset
+
+        :param prefix str: Path to the prefix
+        :param source str: name of the original asset
+        :param destination str: name of the new asset
+        """
+
+        if self == self.UNKNOWN:
+            raise RuntimeError("Trying to fork an asset of unknown type")
+        success = common.fork(prefix, self.name.lower(), source, destination)
+        return success == 0
+
+    def delete(self, prefix, name):
+        """Delete an asset
+
+        :param prefix str: Path to the prefix
+        :param name str: name of the asset to delete
+        """
+
+        if self == self.UNKNOWN:
+            raise RuntimeError("Trying to delete an asset of unknown type")
+        success = common.delete_local(prefix, self.name.lower(), [name])
+        return success == 0
+
+    def has_code(self):
+        """Returns whether this asset type contains code"""
+
+        if self.klass is None:
+            return False
+
+        return issubclass(self.storage, utils.CodeStorage)
+
+
+class Asset:
+    """Class encapsulating an asset"""
+
+    def __init__(self, prefix, asset_type, asset_name):
+        self.prefix = prefix
+        self.type = asset_type
+        self.name = asset_name
+
+    def __eq__(self, other):
+        """Comparison operator"""
+
+        if isinstance(other, Asset):
+            return (
+                self.type == other.type
+                and self.name == other.name
+                and self.prefix == other.prefix
+            )
+        return False
+
+    def __repr__(self):
+        """Representation"""
+
+        return f"{self.type}: {self.name}"
+
+    @staticmethod
+    def from_path(prefix, path):
+        """Builds an asset based on a full path with the given prefix"""
+
+        asset_path = path[len(prefix) :]  # noqa
+        asset_str = asset_path.split("/")[1]
+        asset_type = AssetType.from_path(asset_str)
+        asset_name = "/".join(asset_path.split("/")[2:]).split(".")[0]
+        return Asset(prefix, asset_type, asset_name)
+
+    @property
+    def declaration_path(self):
+        """Returns the full path to the declaration file"""
+
+        if self.type == AssetType.UNKNOWN:
+            raise RuntimeError("Trying to get declaration of unknown type")
+
+        return self.storage().json.path
+
+    @property
+    def declaration(self):
+        """Returns the JSON content loaded from the file"""
+
+        with open(self.declaration_path, "rt") as json_file:
+            return json.load(json_file)
+
+    @declaration.setter
+    def declaration(self, declaration):
+        with open(self.declaration_path, "wt") as json_file:
+            json_file.write(
+                json.dumps(
+                    declaration, sort_keys=True, indent=4, cls=utils.NumpyJSONEncoder
+                )
+            )
+
+    @property
+    def documentation_path(self):
+        """Returns the full path to the documentation file"""
+
+        if self.type == AssetType.UNKNOWN:
+            raise RuntimeError("Trying to get documentation of unknown type")
+
+        return self.storage().doc.path
+
+    @property
+    def code_path(self):
+        """Returns the full path to the code file"""
+
+        if self.type == AssetType.UNKNOWN:
+            raise RuntimeError("Trying to get code of unknown type")
+
+        if not self.type.has_code():
+            return None
+
+        return self.storage().code.path
+
+    def delete(self):
+        """Deletes the asset pointed to by this object"""
+
+        return self.type.delete(self.prefix, self.name)
+
+    def storage(self):
+        """Returns the storage object for this asset"""
+
+        return self.type.storage(self.prefix, self.name)
+
+    def is_valid(self):
+        """Returns whether the declaration of this asset is valid and the list
+        of error associated.
+        """
+
+        _, error_list = self.type.validate(self.declaration_path)
+        return len(error_list) == 0, error_list

beat/editor/backend/assetmodel.py

+# vim: set fileencoding=utf-8 :
+###############################################################################
+#                                                                             #
+# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/           #
+# Contact: beat.support@idiap.ch                                              #
+#                                                                             #
+# This file is part of the beat.editor module of the BEAT platform.           #
+#                                                                             #
+# Commercial License Usage                                                    #
+# Licensees holding valid commercial BEAT licenses may use this file in       #
+# accordance with the terms contained in a written agreement between you      #
+# and Idiap. For further information contact tto@idiap.ch                     #
+#                                                                             #
+# Alternatively, this file may be used under the terms of the GNU Affero      #
+# Public License version 3 as published by the Free Software and appearing    #
+# in the file LICENSE.AGPL included in the packaging of this file.            #
+# The BEAT platform is distributed in the hope that it will be useful, but    #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
+#                                                                             #
+# You should have received a copy of the GNU Affero Public License along      #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
+#                                                                             #
+###############################################################################
+
+import os
+
+from PyQt5.QtCore import pyqtSignal
+from PyQt5.QtCore import pyqtSlot
+from PyQt5.QtCore import pyqtProperty
+from PyQt5.QtCore import QStringListModel
+
+from ..utils import dataformat_basetypes
+
+from .asset import AssetType
+from .asset import Asset
+
+
+class AssetModel(QStringListModel):
+    """The asset model present a list of available asset from a given type"""
+
+    assetTypeChanged = pyqtSignal(AssetType)
+    prefixPathChanged = pyqtSignal(str)
+
+    def __init__(self, parent=None):
+        """Constructor"""
+
+        super().__init__(parent)
+
+        self.__latest_only = True
+        self.__prefix_path = None
+        self.__asset_type = AssetType.UNKNOWN
+
+    def setLatestOnlyEnabled(self, enabled):
+        if self.__latest_only == enabled:
+            return
+        self.__latest_only = enabled
+        self.reload()
+
+    @pyqtSlot()
+    def reload(self):
+        """Loads the content regarding the asset property from the prefix"""
+
+        if not self.__prefix_path or self.__asset_type == AssetType.UNKNOWN:
+            return
+
+        def _find_json_files(path):
+            """Return all json files from folder sorted"""
+
+            asset_items = os.scandir(path)
+            json_files = sorted(
+                [
+                    item.name
+                    for item in asset_items
+                    if item.is_file() and item.name.endswith("json")
+                ]
+            )
+            return json_files
+
+        assets_list = []
+
+        if self.asset_type in [AssetType.DATABASE, AssetType.PROTOCOLTEMPLATE]:
+            # These assets have no user associated with them
+            asset_folders = [
+                entry for entry in os.scandir(self.asset_folder) if entry.is_dir()
+            ]
+            for asset_folder in asset_folders:
+                json_files = _find_json_files(asset_folder)
+                if json_files:
+                    if self.__latest_only:
+                        json_files = json_files[-1:]
+
+                    for json_file in json_files:
+                        assets_list.append(
+                            "{name}/{version}".format(
+                                name=asset_folder.name, version=json_file.split(".")[0]
+                            )
+                        )
+        else:
+            # Assets belonging to a user
+            asset_users = [
+                entry for entry in os.scandir(