Commit 468f44ee authored by Philip ABBET's avatar Philip ABBET

Merge branch 'docker' into 'master'

parents 6602a993 97d9afaf
py27-linux:
script:
- git clean -ffdx
- export TMPDIR=/var/tmp
- /idiap/project/beat/environments/staging/usr/bin/python bootstrap-buildout.py --setuptools-version=`/idiap/project/beat/environments/staging/usr/bin/python -c 'import setuptools; print(setuptools.__version__)'`
- ./bin/buildout
- ./bin/python --version
- unset TMPDIR
- export NOSE_WITH_COVERAGE=1
- export NOSE_COVER_PACKAGE=beat.cmdline
- ./bin/nosetests -sv
- ./bin/sphinx-apidoc --separate -d 2 --output=doc/api beat
- ./bin/sphinx-build doc html
tags:
- lidiap2015
stages:
- build
variables:
PREFIX: /opt/beat.env.web-${CI_BUILD_REF_NAME}/usr
py27-macosx:
build:
stage: build
before_script:
- ${PREFIX}/bin/python --version
- docker info
script:
- git clean -ffdx
- /Users/buildbot/work/environments/beat/py27/bin/python bootstrap-buildout.py --setuptools-version=`/Users/buildbot/work/environments/beat/py27/bin/python -c 'import setuptools; print(setuptools.__version__)'`
- ${PREFIX}/bin/python bootstrap-buildout.py
- ./bin/buildout
- ./bin/python --version
- export NOSE_WITH_COVERAGE=1
- export NOSE_COVER_PACKAGE=beat.cmdline
- ./bin/nosetests -sv
- ./bin/sphinx-apidoc --separate -d 2 --output=doc/api beat
- ./bin/sphinx-build doc html
- ./bin/python ${PREFIX}/bin/coverage run --source=${CI_PROJECT_NAME} ${PREFIX}/bin/nosetests -sv ${CI_PROJECT_NAME}
- ./bin/python ${PREFIX}/bin/coverage report
- ./bin/python ${PREFIX}/bin/sphinx-apidoc --separate -d 2 --output=doc/api ${CI_PROJECT_NAMESPACE}
- ./bin/python ${PREFIX}/bin/sphinx-build doc html
tags:
- beat-macosx
- docker-build
\ No newline at end of file
......@@ -27,7 +27,7 @@
"""Usage:
%(prog)s experiments run [--env=<path>...] [--force] <name>
%(prog)s experiments run [--force] <name>
%(prog)s experiments caches [--list | --delete | --checksum] <name>
%(prog)s experiments list [--remote]
%(prog)s experiments check [<name>]...
......@@ -66,9 +66,6 @@ Options:
--help Display this screen
--path=<dir> Use path to write files to disk (instead of the current
directory)
--env=<path> If set, use this path to search for execution environments
for blocks. Otherwise, execute each block using the built-in
executor and the currently installed (python) environment.
"""
......@@ -83,12 +80,13 @@ import simplejson
from . import common
from beat.core.experiment import Experiment
from beat.core.execution import Executor, discover_environments
from beat.core.execution import Executor
from beat.core.utils import NumpyJSONEncoder
from beat.core.data import CachedDataSource, load_data_index
from beat.core.dock import Host
def run_experiment(configuration, name, force, envpaths):
def run_experiment(configuration, name, force):
'''Run experiments locally'''
def load_result(executor):
......@@ -163,12 +161,9 @@ def run_experiment(configuration, name, force, envpaths):
scheduled = experiment.setup()
# load existing environments
environments = {}
if envpaths:
environments.update(discover_environments(envpaths,
raise_on_errors=not force))
logger.info("Discovered %d environments under `%s'",
len(environments), os.pathsep.join(envpaths))
host = Host()
host.setup(raise_on_errors=False)
environments = host.environments
# can we execute it?
results = []
......@@ -177,18 +172,13 @@ def run_experiment(configuration, name, force, envpaths):
# checks and sets-up executable
executable = None #use the default
if envpaths: #user wants to execute block on the adequate environment
env = value['configuration']['environment']
search_key = '%s (%s)' % (env['name'], env['version'])
if search_key not in environments:
logger.error("Cannot execute block `%s' on environment `%s': " \
"environment was not found on `%s' - please install it",
key, search_key, os.pathsep.join(envpaths))
return 1
executable = environments[search_key]['execute']
if executable is not None:
executable = os.path.join(os.path.dirname(executable), 'execute')
env = value['configuration']['environment']
search_key = '%s (%s)' % (env['name'], env['version'])
if search_key not in environments:
logger.error("Cannot execute block `%s' on environment `%s': " \
"environment was not found' - please install it",
key, search_key)
return 1
executor = Executor(configuration.path, value['configuration'],
configuration.cache, dataformat_cache, database_cache,
......@@ -211,7 +201,7 @@ def run_experiment(configuration, name, force, envpaths):
logger.extra(" -> using fallback (default) environment")
with executor:
result = executor.process(executable)
result = executor.process(host)
if result['status'] != 0:
logger.error("Block did not execute properly - outputs were reset")
......@@ -384,8 +374,7 @@ def pull(webapi, prefix, names, force, indentation, format_cache):
def process(args):
if args['run']:
return run_experiment(args['config'], args['<name>'][0], args['--force'],
args['--env'])
return run_experiment(args['config'], args['<name>'][0], args['--force'])
if args['caches']:
return caches(args['config'], args['<name>'][0], args['--list'],
......
......@@ -199,15 +199,6 @@ def test_run_single_error_twice():
nose.tools.eq_(call('run', obj, cache=tmp_prefix), 1)
@slow
@nose.tools.with_setup(teardown=cleanup)
def test_run_with_environment():
obj = 'user/user/integers_addition/1/integers_addition'
D = os.path.dirname
envpath = D(D(D(os.path.realpath(sys.argv[0]))))
nose.tools.eq_(call('run', obj, '--force', '--env=%s' % envpath, cache=tmp_prefix), 0)
@nose.tools.with_setup(teardown=cleanup)
@skipif(disconnected, "missing test platform (%s)" % platform)
def test_push_and_delete():
......
......@@ -3,7 +3,7 @@
; Tue 02 Jun 2015 13:05:59 CEST
[buildout]
parts = scripts cpulimit
parts = scripts
extensions = mr.developer
auto-checkout = *
develop = .
......@@ -13,17 +13,8 @@ eggs = beat.cmdline
ipdb
[sources]
beat.core = git git@gitlab.idiap.ch:beat/beat.core
beat.backend.python = git git@gitlab.idiap.ch:beat/beat.backend.python
cpulimit = git https://github.com/opsengine/cpulimit rev=v0.2 egg=false
[cpulimit]
recipe = collective.recipe.cmd
cmds = make -C src/cpulimit
cd bin && ln -s ../src/cpulimit/src/cpulimit && cd ..
uninstall_cmds = rm -f bin/cpulimit
on_install = true
on_update = true
beat.core = git https://gitlab.idiap.ch/beat/beat.core
beat.backend.python = git https://gitlab.idiap.ch/beat/beat.backend.python
[scripts]
recipe = bob.buildout:scripts
......@@ -27,12 +27,12 @@ Algorithms
The commands available for algorithms are:
.. command-output:: ./bin/beat algorithms --help
:cwd:..
:cwd: ..
For instance, a list of the algorithms available on the remote platform can
be obtained as follows:
.. command-output:: ./bin/beat algorithms list --remote
:cwd:..
:cwd: ..
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment