Commit 99ac5da0 authored by André Anjos's avatar André Anjos 💬
Browse files

Merge branch 'conda-ci' into 'master'

Support for new conda-based CI/CD pipelines

Closes #91

See merge request !3
parents d1beb62a 953e02f9
Pipeline #19832 failed with stages
in 50 minutes and 54 seconds
......@@ -78,3 +78,38 @@ user_conf.json
# pycache
__pycache__
*~
*.swp
*.pyc
*.so
bin
eggs
parts
.installed.cfg
.mr.developer.cfg
*.egg-info
develop-eggs
sphinx
doc/api
src
dist
.nfs*
.gdb_history
build
*.egg
opsnr.stt
.coverage
.DS_Store
html/
record.txt
_ci/
miniconda.sh
miniconda/
miniconda.cached/
conda/recipe_append.yaml
conda-bld/
# built JS files
beat/editor/js
prefix/
# This build file uses template features from YAML so it is generic enough for
# any Bob project. Don't modify it unless you know what you're doing.
# Definition of global variables (all stages)
variables:
CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
# Definition of our build pipeline order
stages:
- build
- browser-tests
- deploy
- pypi
# Build targets
.build_template: &build_job
stage: build
before_script:
- mkdir _ci
- curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
- chmod 755 _ci/install.sh
- ./_ci/install.sh _ci master #installs ci support scripts
- ./_ci/before_build.sh
script:
- ./_ci/build.sh
after_script:
- ./_ci/after_build.sh
cache: &build_caches
paths:
- miniconda.sh
- ${CONDA_ROOT}/pkgs/*.tar.bz2
- ${CONDA_ROOT}/pkgs/urls.txt
.build_linux_template: &linux_build_job
<<: *build_job
tags:
- docker
image: continuumio/conda-concourse-ci
artifacts:
expire_in: 1 week
paths:
- _ci/
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
cache:
<<: *build_caches
key: "linux-cache"
.build_macosx_template: &macosx_build_job
<<: *build_job
tags:
- macosx
artifacts:
expire_in: 1 week
paths:
- _ci/
- ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
cache:
<<: *build_caches
key: "macosx-cache"
build_linux_27:
<<: *linux_build_job
variables:
PYTHON_VERSION: "2.7"
build_linux_36:
<<: *linux_build_job
variables:
PYTHON_VERSION: "3.6"
BUILD_EGG: "true"
artifacts:
expire_in: 1 week
paths:
- _ci/
- dist/*.zip
- sphinx
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
build_macosx_27:
<<: *macosx_build_job
variables:
PYTHON_VERSION: "2.7"
build_macosx_36:
<<: *macosx_build_job
variables:
PYTHON_VERSION: "3.6"
# Docker host based testing (must be run inside dind or docker-enabled host)
.browser_test_linux_template: &linux_browser_test_job
stage: browser-tests
image: docker.idiap.ch/beat/beat.env.editor:0.0.1r5
before_script:
# safe keep artifacts as before_build.sh will erase those...
- mv ${CONDA_ROOT}/conda-bld .
- ./_ci/install.sh _ci master #updates ci support scripts
- ./_ci/before_build.sh
- mv conda-bld ${CONDA_ROOT}
script:
- export BEAT_BROWSER_TESTS=true
- BOB_TEST_ONLY=true ./_ci/build.sh
after_script:
- ./_ci/after_build.sh
browser_linux_36:
<<: *linux_browser_test_job
variables:
PYTHON_VERSION: "3.6"
dependencies:
- build_linux_36
tags:
- docker
# Deploy targets
.deploy_template: &deploy_job
stage: deploy
before_script:
- ./_ci/install.sh _ci master #updates ci support scripts
script:
- ./_ci/deploy.sh
dependencies:
- build_linux_27
- build_linux_36
- build_macosx_27
- build_macosx_36
tags:
- deployer
deploy_beta:
<<: *deploy_job
environment: beta
only:
- master
deploy_stable:
<<: *deploy_job
environment: stable
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
pypi:
stage: pypi
environment: pypi
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
before_script:
- ./_ci/install.sh _ci master #updates ci support scripts
script:
- ./_ci/pypi.sh
dependencies:
- build_linux_36
tags:
- deployer
This diff is collapsed.
include LICENSE.AGPL README.rst version.txt requirements.txt buildout.cfg
recursive-include doc conf.py *.rst *.png *.ico
recursive-include beat/editor/templates *.jinja2
recursive-include beat/editor/js *
.. vim: set fileencoding=utf-8 :
.. Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ ..
.. Contact: beat.support@idiap.ch ..
.. ..
.. This file is part of the beat.editor module of the BEAT platform. ..
.. ..
.. Commercial License Usage ..
.. Licensees holding valid commercial BEAT licenses may use this file in ..
.. accordance with the terms contained in a written agreement between you ..
.. and Idiap. For further information contact tto@idiap.ch ..
.. ..
.. Alternatively, this file may be used under the terms of the GNU Affero ..
.. Public License version 3 as published by the Free Software and appearing ..
.. in the file LICENSE.AGPL included in the packaging of this file. ..
.. The BEAT platform is distributed in the hope that it will be useful, but ..
.. WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY ..
.. or FITNESS FOR A PARTICULAR PURPOSE. ..
.. ..
.. You should have received a copy of the GNU Affero Public License along ..
.. with the BEAT platform. If not, see http://www.gnu.org/licenses/. ..
.. image:: https://img.shields.io/badge/docs-stable-yellow.svg
:target: https://www.idiap.ch/software/beat/docs/beat/beat.editor/stable/index.html
.. image:: https://img.shields.io/badge/docs-latest-orange.svg
:target: https://www.idiap.ch/software/beat/docs/beat/beat.editor/master/index.html
.. image:: https://gitlab.idiap.ch/beat/beat.editor/badges/master/build.svg
:target: https://gitlab.idiap.ch/beat/beat.editor/commits/master
.. image:: https://gitlab.idiap.ch/beat/beat.editor/badges/master/coverage.svg
:target: https://gitlab.idiap.ch/beat/beat.editor/commits/master
.. image:: https://img.shields.io/badge/gitlab-project-0000c0.svg
:target: https://gitlab.idiap.ch/beat/beat.editor
.. image:: https://img.shields.io/pypi/v/beat.editor.svg
:target: https://pypi.python.org/pypi/beat.editor
===============================
Local editor for BEAT objects
===============================
This package part of BEAT_, an open-source evaluation platform for data science
algorithms and workflows. It contains the source code for a local editor for
BEAT objects.
Installation
------------
Complete BEAT's `installation`_ instructions. Then, to install this package,
run::
$ conda install beat.editor
Contact
-------
For questions or reporting issues to this software package, contact our
development `mailing list`_.
.. Place your references here:
.. _beat: https://www.idiap.ch/software/beat
.. _installation: https://www.idiap.ch/software/beat/install
.. _mailing list: https://www.idiap.ch/software/beat/discuss
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.editor module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
"""
A simple Flask server providing an API for the local BEAT prefix and user configuration.
Dependencies:
simplejson
flask
flask_restful
flask_cors
"""
#!/usr/bin/env python
# -*- coding: utf-8 -*-
###############################################################################
# #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.editor module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
'''Server resources (API endpoints)'''
import os
import glob
from shutil import copy as copy_file
import shutil
import subprocess
import json
from enum import Enum
import simplejson
from flask import Flask, request
from flask_restful import Resource, Api
from flask_cors import CORS
import templates
app = Flask(__name__)
api = Api(app)
CORS(app)
def get_user_conf():
"""Reads & returns the user configuration in a dict"""
user_conf = {}
if os.path.isfile('./user_conf.json'):
with open('./user_conf.json') as f:
try:
user_conf = json.load(f)
except json.JSONDecodeError:
print('user_conf.json is invalid!')
raise SystemExit
else:
with open('./user_conf.json', 'w') as f:
user_conf = {
'prefix': '~'
}
json.dump(user_conf, f)
if 'prefix' not in user_conf:
raise Exception('Invalid user_conf: Needs "prefix" key!')
return user_conf
def get_prefix():
"""Returns an absolute path to the user-defined BEAT prefix location"""
return os.path.expanduser(get_user_conf()['prefix'])
def get_tc_layout(tc_name):
"""Returns the JSON returned by Graphviz's dot "-Tjson0" output for the given toolchain"""
prefix = get_prefix()
beat_ex_loc = '%s/bin/beat' % prefix
tc_dot_loc = '%s/toolchains/%s.dot' % (prefix, tc_name)
if not os.path.isfile(beat_ex_loc):
raise Exception('BEAT executable not at %s' % beat_ex_loc)
beat_cmd = 'cd %s && ./bin/beat tc draw %s' % (prefix, tc_name)
output = subprocess.call(beat_cmd, shell=True)
if not os.path.isfile(tc_dot_loc):
print('Running command "%s" got:' % beat_cmd)
print(output)
raise Exception('"%s" graphviz dot file not found at "%s"' % (tc_name, tc_dot_loc))
s = subprocess.check_output('dot %s -Tjson0' % tc_dot_loc, shell=True, encoding='utf-8')
return s
def get_environment_info():
json = simplejson.loads('''
[
{
"name": "Scientific Python 2.7",
"packages": {
"alabaster": "0.7.6",
"babel": "1.3"
},
"queues": {
"Default": {
"memory_limit": 5983,
"nb_slots": 2,
"max_slots_per_user": 2,
"nb_cores_per_slot": 1,
"time_limit": 360
}
},
"accessibility": "public",
"languages": [
"python"
],
"version": "0.0.4",
"short_description": "Scientific Python 2.7"
},
{
"name": "Scientific Python 2.7",
"packages": {
"alabaster": "0.7.10",
"Babel": "2.4.0"
},
"queues": {
"Default": {
"memory_limit": 5983,
"nb_slots": 2,
"max_slots_per_user": 2,
"nb_cores_per_slot": 1,
"time_limit": 360
}
},
"accessibility": "public",
"languages": [
"python"
],
"version": "1.0.0",
"short_description": "Scientific Python 2.7"
}
]
''')
return json
from flask import request
from flask_restful import Resource
import logging
logger = logging.getLogger(__name__)
from . import utils
from beat.core.dock import Host
from beat.core.environments import enumerate_packages
class Layout(Resource):
"""Exposes toolchain layout functionality"""
def __init__(self, config):
self.config = config
def post(self):
data = request.get_json()
if data != None and 'toolchain' in data:
from beat.core.toolchain import Toolchain
obj = Toolchain(self.config.path, data['toolchain'])
diagram = obj.dot_diagram(is_layout=True)
diagram.format = 'json'
return diagram.pipe().decode()
else:
raise RuntimeError('Invalid post content for tc layout!')
class Environments(Resource):
"""Exposes local environment info"""
def get(self):
"""Uses beat.core to get the local environment (docker) information
and returns a list of environments"""
host = Host()
envs = host.processing_environments
for env in envs.keys():
envs[env]['queues'] = {}
try:
envs[env]['packages'] = enumerate_packages(host, env)
except:
envs[env]['packages'] = []
return envs
class BeatEntity(Enum):
"""List of BEAT entities to serve to the webapp"""
DATAFORMAT = 'dataformats'
DATABASE = 'databases'
LIBRARY = 'libraries'
ALGORITHM = 'algorithms'
TOOLCHAIN = 'toolchains'
EXPERIMENT = 'experiments'
PLOTTER = 'plotters'
PLOTTERPARAMETER = 'plotterparameters'
class Templates(Resource):
"""Endpoint for generating template files"""
def __init__(self, config):
self.config = config
def read_json(path):
"""Reads a JSON file and returns the parse JSON obj"""
with open(path, 'r') as f:
data = json.load(f)
return data
def post(self):
data = request.get_json()
entity = data.pop('entity')
name = data.pop('name')
utils.generate_python_template(entity, name, self.config, **data)
def path_to_dict(path):
"""Generates a dict of the given file/folder in the BEAT prefix"""
d = {
'name': os.path.basename(path)
}
d = dict(name=os.path.basename(path))
if os.path.isdir(path):
d['type'] = "directory"
d['children'] = [path_to_dict(os.path.join(path, x))
......@@ -151,17 +109,39 @@ def path_to_dict(path):
d['type'] = "file"
fname, fext = os.path.splitext(path)
if fext == '.json':
d['json'] = read_json(path)
with open(path, 'rt') as f:
d['json'] = simplejson.loads(f.read())
return d
def generate_file_tree(be):
VALID_ENTITIES = [
'dataformats',
'databases',
'libraries',
'algorithms',
'toolchains',
'experiments',
'plotters',
'plotterparameters',
]
"""List of valid BEAT object entitities"""
def assert_valid_entity(v):
"""Asserts the passed value corresponds to a valid BEAT entity"""
assert v in VALID_ENTITIES, '%s is not a valid BEAT entity ' \
'(valid values are %s)' % (v, ', '.join(VALID_ENTITIES))
def generate_file_tree(entity, config):
"""Generates a file tree (of dicts) given a specific BEAT entity"""
resource_path = os.path.join(get_prefix(), be.value)
assert_valid_entity(entity)
resource_path = os.path.join(config.path, entity)
if not os.path.isdir(resource_path):
raise NotADirectoryError('Invalid resource path %s' % resource_path)
raise IOError('Invalid resource path %s' % resource_path)
#print('generating entity tree for path %s' % resource_path)
return path_to_dict(resource_path)
......@@ -184,20 +164,21 @@ def generate_json_entity(fto, parent_names):
}
def generate_entity_tree(be):
def generate_entity_tree(entity, config):
"""Generates the entire tree for an entity type from the prefix"""
file_tree = generate_file_tree(be)
file_tree = generate_file_tree(entity, config)
entity_tree = {}
user_and_name = [
BeatEntity.DATAFORMAT,
BeatEntity.LIBRARY,
BeatEntity.ALGORITHM,
BeatEntity.TOOLCHAIN,
BeatEntity.PLOTTER,
BeatEntity.PLOTTERPARAMETER,
'dataformats',
'libraries',
'algorithms',
'toolchains',
'plotters',
'plotterparameters',
]
if be in user_and_name:
if entity in user_and_name:
for user in file_tree['children']:
entity_tree[user['name']] = {}