Commit 849d0867 authored by Flavio TARSETTI's avatar Flavio TARSETTI

Merge branch 'v2' into 'master'

V2

Closes #244, #226, #242, #232, #181, #234, #233, #231, #223, #222, #221, #215, #214, #183, #212, #178, #204, #191, #189, and #190

See merge request !112
parents 78957dca be34e030
Pipeline #34407 passed with stage
in 17 minutes and 33 seconds
[flake8]
max-line-length = 80
select = B,C,E,F,W,T4,B9,B950
ignore = E501, W503, E203
# Created by https://www.gitignore.io/api/node
### Node ###
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
# Runtime data
pids
*.pid
*.seed
*.pid.lock
# Directory for instrumented libs generated by jscoverage/JSCover
lib-cov
# Coverage directory used by tools like istanbul
coverage
# nyc test coverage
.nyc_output
# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
.grunt
# Bower dependency directory (https://bower.io/)
bower_components
# node-waf configuration
.lock-wscript
# Compiled binary addons (http://nodejs.org/api/addons.html)
build/Release
# Dependency directories
node_modules/
jspm_packages/
# Typescript v1 declaration files
typings/
# Optional npm cache directory
.npm
# Optional eslint cache
.eslintcache
# Optional REPL history
.node_repl_history
# Output of 'npm pack'
*.tgz
# Yarn Integrity file
.yarn-integrity
# dotenv environment variables file
.env
# End of https://www.gitignore.io/api/node
# built files
dist/
# docs cache/build
styleguide/
flow-typed/
coverage/
# user conf file for app
user_conf.json
# pycache
__pycache__
......@@ -109,8 +31,8 @@ miniconda.cached/
conda/recipe_append.yaml
conda-bld/
# built JS files
beat/editor/js
# users prefix
prefix/
conda/js/package.json
conda/js/webpack.config.js
# development buildout
src/
include: 'https://gitlab.idiap.ch/bob/bob.devtools/raw/master/bob/devtools/data/gitlab-ci/single-package.yaml'
# Redefines the pipeline order for this package only
stages:
- build
- browser-tests
- deploy
- pypi
# Docker host based testing (must be run inside dind or docker-enabled host)
browser_test:
extends: .test_linux_template
stage: browser-tests
image: docker.idiap.ch/beat/ci.env.editor:0.0.1r0
variables:
BEAT_BROWSER_TESTS: "true"
LANG: "en_US.UTF-8"
LANGUAGE: "en_US:en"
LC_ALL: "en_US.UTF-8"
dependencies:
- build_linux_36
before_script:
- curl --silent "${BOOTSTRAP}" --output "bootstrap.py"
- python3 bootstrap.py -vv channel base
- source ${CONDA_ROOT}/etc/profile.d/conda.sh
- conda activate base
- apt-get update > /dev/null
- apt-get install -y locales > /dev/null
- echo "en_US UTF-8" > /etc/locale.gen
- locale-gen en_US.UTF-8
- rm -rf ${CI_PROJECT_DIR}/sphinx
......@@ -17,4 +17,4 @@ Relevant links / references
(Paste any relevant links / image for the implementation of the suggestion)
/label ~suggestion
\ No newline at end of file
/label ~suggestion
# See https://pre-commit.com for more information
# See https://pre-commit.com/hooks.html for more hooks
repos:
- repo: https://github.com/ambv/black
rev: stable
hooks:
- id: black
language_version: python3.6
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v2.0.0
hooks:
- id: trailing-whitespace
- id: end-of-file-fixer
- id: debug-statements
- id: check-added-large-files
- id: check-docstring-first
- id: flake8
- id: check-yaml
exclude: conda/meta.yaml
- repo: https://github.com/PyCQA/bandit
rev: 'master' # Update me!
hooks:
- id: bandit
exclude: beat/editor/test
- repo: local
hooks:
- id: sphinx-build
name: sphinx build
entry: python -m sphinx.cmd.build
args: [-a, -E, -W, doc, sphinx]
language: system
files: ^doc/
types: [file]
pass_filenames: false
- id: sphinx-doctest
name: sphinx doctest
entry: python -m sphinx.cmd.build
args: [-a, -E, -b, doctest, doc, sphinx]
language: system
files: ^doc/
types: [file]
pass_filenames: false
include LICENSE.AGPL README.rst version.txt requirements.txt buildout.cfg
recursive-include doc conf.py *.rst *.png *.ico
recursive-include doc conf.py *.rst *.png *.ico *.md
recursive-include beat/editor/templates *.jinja2
recursive-include beat/editor/js *
......@@ -27,4 +27,5 @@
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.editor module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
......@@ -24,39 +24,3 @@
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
# test the resources.py file
# (mostly endpoints and working with the filesystem)
import nose.tools
import os
from .. import resources
# the func names the endpoint the given name
def test_check_valid_generated_endpoint_name():
name = 'TestEndpoint'
endpoint = resources.gen_endpoint(name)
nose.tools.eq_(endpoint.__name__, name)
# the func doesnt accept non-entity names
@nose.tools.raises(AssertionError)
def test_assert_valid_entity_invalid():
resources.assert_valid_entity('notanentity')
# the func parses this file
def test_path_to_dict_file():
currfile = os.path.realpath(__file__)
res = resources.path_to_dict(currfile)
# in python 3 the first case works but in python 2 the second case works
assert res == {'name': 'test_resources.py', 'type': 'file'} or res == {'name': 'test_resources.pyc', 'type': 'file'}
# the func parses this folder
def test_path_to_dict_folder():
currfolder = os.path.dirname(os.path.realpath(__file__))
res = resources.path_to_dict(currfolder)
nose.tools.eq_(res['name'], 'test')
nose.tools.eq_(res['type'], 'directory')
nose.tools.ok_({'name': '__init__.py', 'type': 'file'} in res['children'])
nose.tools.ok_({'name': 'test_resources.py', 'type': 'file'} in res['children'])
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.editor module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
import importlib
import simplejson as json
import beat.core
from enum import Enum, unique
from beat.backend.python import utils
from beat.core.schema import validate
from beat.cmdline import common
@unique
class AssetType(Enum):
"""All possible assets available on the BEAT platform"""
UNKNOWN = ("unknown", None)
ALGORITHM = ("algorithms", beat.core.algorithm.Algorithm)
DATABASE = ("databases", beat.core.database.Database)
DATAFORMAT = ("dataformats", beat.core.dataformat.DataFormat)
EXPERIMENT = ("experiments", beat.core.experiment.Experiment)
LIBRARY = ("libraries", beat.core.library.Library)
PLOTTER = ("plotters", beat.core.plotter.Plotter)
PLOTTERPARAMETER = (
"plotterparameters",
beat.core.plotterparameter.Plotterparameter,
)
PROTOCOLTEMPLATE = (
"protocoltemplates",
beat.core.protocoltemplate.ProtocolTemplate,
)
TOOLCHAIN = ("toolchains", beat.core.toolchain.Toolchain)
def __init__(self, path, klass):
self.path = path
self.klass = klass
@property
def storage(self):
mod = importlib.import_module(self.klass.__module__)
return getattr(mod, "Storage")
@staticmethod
def from_path(path):
for asset_type in AssetType:
if asset_type.path == path:
return asset_type
raise RuntimeError("Unknown asset path {}".format(path))
def can_create(self):
"""Returns whether a new asset can be created from scratch"""
return self not in [self.UNKNOWN, self.EXPERIMENT]
def has_versions(self):
"""Returns whether a new version of this asset can be created"""
return self not in [self.UNKNOWN, self.EXPERIMENT]
def can_fork(self):
"""Returns whether a new asset can be forked"""
return self not in [self.UNKNOWN, self.DATABASE, self.PROTOCOLTEMPLATE]
def split_count(self):
"""Returns the number of "/" that should be part of its name"""
if self == self.UNKNOWN:
return 0
elif self == self.EXPERIMENT:
return 5
elif self not in [self.DATABASE, self.PROTOCOLTEMPLATE]:
return 2
else:
return 1
def validate(self, data):
"""Runs the schema validation and returns whether an asset is valid
:param data str: asset content
"""
if self == self.UNKNOWN:
raise RuntimeError("Trying to validate unknown type")
return validate(self.name.lower(), data)
def create_new(self, prefix, name):
"""Create a new asset from a prototype
:param prefix str: Path to the prefix
:param name str: name of the asset
"""
if self == self.UNKNOWN:
raise RuntimeError("Trying to create an asset of unknown type")
success = common.create(prefix, self.name.lower(), [name])
return success == 0
def create_new_version(self, prefix, name):
"""Create a new version of the asset
:param prefix str: Path to the prefix
:param name str: name of the asset
"""
if self == self.UNKNOWN:
raise RuntimeError(
"Trying to create a new version of an asset of unknown type"
)
success = common.new_version(prefix, self.name.lower(), name)
return success == 0
def fork(self, prefix, source, destination):
"""Fork an asset
:param prefix str: Path to the prefix
:param source str: name of the original asset
:param destination str: name of the new asset
"""
if self == self.UNKNOWN:
raise RuntimeError("Trying to fork an asset of unknown type")
success = common.fork(prefix, self.name.lower(), source, destination)
return success == 0
def delete(self, prefix, name):
"""Delete an asset
:param prefix str: Path to the prefix
:param name str: name of the asset to delete
"""
if self == self.UNKNOWN:
raise RuntimeError("Trying to delete an asset of unknown type")
success = common.delete_local(prefix, self.name.lower(), [name])
return success == 0
def has_code(self):
"""Returns whether this asset type contains code"""
if self.klass is None:
return False
return issubclass(self.storage, utils.CodeStorage)
class Asset:
"""Class encapsulating an asset"""
def __init__(self, prefix, asset_type, asset_name):
self.prefix = prefix
self.type = asset_type
self.name = asset_name
def __eq__(self, other):
"""Comparison operator"""
if isinstance(other, Asset):
return (
self.type == other.type
and self.name == other.name
and self.prefix == other.prefix
)
return False
def __repr__(self):
"""Representation"""
return f"{self.type}: {self.name}"
@staticmethod
def from_path(prefix, path):
"""Builds an asset based on a full path with the given prefix"""
asset_path = path[len(prefix) :] # noqa
asset_str = asset_path.split("/")[1]
asset_type = AssetType.from_path(asset_str)
asset_name = "/".join(asset_path.split("/")[2:]).split(".")[0]
return Asset(prefix, asset_type, asset_name)
@property
def declaration_path(self):
"""Returns the full path to the declaration file"""
if self.type == AssetType.UNKNOWN:
raise RuntimeError("Trying to get declaration of unknown type")
return self.storage().json.path
@property
def declaration(self):
"""Returns the JSON content loaded from the file"""
with open(self.declaration_path, "rt") as json_file:
return json.load(json_file)
@declaration.setter
def declaration(self, declaration):
with open(self.declaration_path, "wt") as json_file:
json_file.write(
json.dumps(
declaration, sort_keys=True, indent=4, cls=utils.NumpyJSONEncoder
)
)
@property
def documentation_path(self):
"""Returns the full path to the documentation file"""
if self.type == AssetType.UNKNOWN:
raise RuntimeError("Trying to get documentation of unknown type")
return self.storage().doc.path
@property
def code_path(self):
"""Returns the full path to the code file"""
if self.type == AssetType.UNKNOWN:
raise RuntimeError("Trying to get code of unknown type")
if not self.type.has_code():
return None
return self.storage().code.path
def delete(self):
"""Deletes the asset pointed to by this object"""
return self.type.delete(self.prefix, self.name)
def storage(self):
"""Returns the storage object for this asset"""
return self.type.storage(self.prefix, self.name)
def is_valid(self):
"""Returns whether the declaration of this asset is valid and the list
of error associated.
"""
_, error_list = self.type.validate(self.declaration_path)
return len(error_list) == 0, error_list
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.editor module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
import os
from PyQt5.QtCore import pyqtSignal
from PyQt5.QtCore import pyqtSlot
from PyQt5.QtCore import pyqtProperty
from PyQt5.QtCore import QStringListModel
from ..utils import dataformat_basetypes
from .asset import AssetType
from .asset import Asset
class AssetModel(QStringListModel):
"""The asset model present a list of available asset from a given type"""
assetTypeChanged = pyqtSignal(AssetType)
prefixPathChanged = pyqtSignal(str)
def __init__(self, parent=None):
"""Constructor"""
super().__init__(parent)
self.__latest_only = True
self.__prefix_path = None
self.__asset_type = AssetType.UNKNOWN
def setLatestOnlyEnabled(self, enabled):
if self.__latest_only == enabled:
return
self.__latest_only = enabled
self.reload()
@pyqtSlot()
def reload(self):
"""Loads the content regarding the asset property from the prefix"""
if not self.__prefix_path or self.__asset_type == AssetType.UNKNOWN:
return
def _find_json_files(path):
"""Return all json files from folder sorted"""
asset_items = os.scandir(path)
json_files = sorted(
[
item.name
for item in asset_items
if item.is_file() and item.name.endswith("json")
]
)
return json_files
assets_list = []
if self.asset_type in [AssetType.DATABASE, AssetType.PROTOCOLTEMPLATE]:
# These assets have no user associated with them
asset_folders = [
entry for entry in os.scandir(self.asset_folder) if entry.is_dir()
]
for asset_folder in asset_folders:
json_files = _find_json_files(asset_folder)
if json_files:
if self.__latest_only:
json_files = json_files[-1:]
for json_file in json_files:
assets_list.append(
"{name}/{version}".format(
name=asset_folder.name, version=json_file.split(".")[0]
)
)
else:
# Assets belonging to a user
asset_users = [
entry for entry in os.scandir(