Commit 2580eac2 authored by André Anjos's avatar André Anjos 💬
Browse files

Merge branch '1.4.x' into 'master'

1.4.x

Closes #40, #34, and #24

See merge request !54
parents d51714a8 76c8bbbf
Pipeline #25454 passed with stages
in 18 minutes and 57 seconds
......@@ -29,4 +29,8 @@ dataformats/
experiments/
libraries/
toolchains/
plotters/
plotterparameters/
.noseids
scripts/_core_docker_pull.sh
_ci/
# This build file uses template features from YAML so it is generic enough for
# any Bob project. Don't modify it unless you know what you're doing.
# Definition of global variables (all stages)
variables:
CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
DOCKER_REGISTRY: docker.idiap.ch
# Definition of our build pipeline order
stages:
- build
- deploy
- pypi
variables:
PREFIX: /opt/beat.env.web/usr
build:
# Build targets
.build_template: &build_job
stage: build
before_script:
- ${PREFIX}/bin/python --version
- docker info
- mkdir _ci
- curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
- chmod 755 _ci/install.sh
- ./_ci/install.sh _ci master #installs ci support scripts
- ./_ci/before_build.sh
- docker info
- ./scripts/docker_pull.sh master #pulls required test images
script:
- ./_ci/build.sh
after_script:
- ./_ci/after_build.sh
cache: &build_caches
paths:
- miniconda.sh
- ${CONDA_ROOT}/pkgs/*.tar.bz2
- ${CONDA_ROOT}/pkgs/urls.txt
.build_linux_template: &linux_build_job
<<: *build_job
tags:
- docker-build
image: continuumio/conda-concourse-ci
artifacts:
expire_in: 1 week
paths:
- _ci/
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
cache:
<<: *build_caches
key: "linux-cache"
.build_macosx_template: &macosx_build_job
<<: *build_job
tags:
- macosx
artifacts:
expire_in: 1 week
paths:
- _ci/
- ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
cache:
<<: *build_caches
key: "macosx-cache"
build_linux_36:
<<: *linux_build_job
variables:
PYTHON_VERSION: "3.6"
BUILD_EGG: "true"
artifacts:
expire_in: 1 week
paths:
- _ci/
- dist/*.zip
- sphinx
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
build_macosx_36:
<<: *macosx_build_job
variables:
PYTHON_VERSION: "3.6"
# Deploy targets
.deploy_template: &deploy_job
stage: deploy
before_script:
- ./_ci/install.sh _ci master #updates ci support scripts
script:
- ./_ci/deploy.sh
dependencies:
- build_linux_36
- build_macosx_36
tags:
- deployer
deploy_beta:
<<: *deploy_job
environment: beta
only:
- master
deploy_stable:
<<: *deploy_job
environment: stable
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
pypi:
stage: pypi
environment: pypi
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
before_script:
- ./_ci/install.sh _ci master #updates ci support scripts
script:
- git clean -ffdx
- ${PREFIX}/bin/python bootstrap-buildout.py
- ./bin/buildout
- ./bin/python ${PREFIX}/bin/coverage run --source=${CI_PROJECT_NAME} ${PREFIX}/bin/nosetests -sv ${CI_PROJECT_NAME}
- ./bin/python ${PREFIX}/bin/coverage report
- ./bin/python ${PREFIX}/bin/sphinx-apidoc --separate -d 2 --output=doc/api beat
- ./bin/python ${PREFIX}/bin/sphinx-build doc html
- ./_ci/pypi.sh
dependencies:
- build_linux_36
tags:
- docker-build
- deployer
include LICENSE.AGPL README.rst buildout.cfg bootstrap-buildout.py
recursive-include doc conf.py *.rst *.png *.svg *.ico *.pdf
include LICENSE.AGPL README.rst version.txt requirements.txt
include buildout.cfg develop.cfg
recursive-include scripts *.sh
recursive-include doc conf.py *.rst *.png *.ico *.pdf
recursive-include beat *.checksum *.index *.json *.data
......@@ -3,7 +3,7 @@
.. Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ ..
.. Contact: beat.support@idiap.ch ..
.. ..
.. This file is part of the beat.cmdline module of the BEAT platform. ..
.. This file is part of the beat.cmdline module of the BEAT platform. ..
.. ..
.. Commercial License Usage ..
.. Licensees holding valid commercial BEAT licenses may use this file in ..
......@@ -20,123 +20,47 @@
.. You should have received a copy of the GNU Affero Public License along ..
.. with the BEAT platform. If not, see http://www.gnu.org/licenses/. ..
.. image:: https://img.shields.io/badge/docs-stable-yellow.svg
:target: https://www.idiap.ch/software/beat/docs/beat/beat.cmdline/stable/index.html
.. image:: https://img.shields.io/badge/docs-latest-orange.svg
:target: https://www.idiap.ch/software/beat/docs/beat/beat.cmdline/master/index.html
.. image:: https://gitlab.idiap.ch/beat/beat.cmdline/badges/master/build.svg
:target: https://gitlab.idiap.ch/beat/beat.cmdline/commits/master
.. image:: https://gitlab.idiap.ch/beat/beat.cmdline/badges/master/coverage.svg
:target: https://gitlab.idiap.ch/beat/beat.cmdline/commits/master
.. image:: https://img.shields.io/badge/gitlab-project-0000c0.svg
:target: https://gitlab.idiap.ch/beat/beat.cmdline
.. image:: https://img.shields.io/pypi/v/beat.cmdline.svg
:target: https://pypi.python.org/pypi/beat.cmdline
============================================
Biometrics Evaluation and Testing Platform
============================================
This package contains the source code for a python-based command-line client
for the BEAT platform.
==========================
Core Components for BEAT
==========================
Dependence Status
-----------------
This package part of BEAT_, an open-source evaluation platform for data science
algorithms and workflows. It contains the command-line based program that can
download/upload components with the platform. It can also run experiments
locally.
Before checking-out sources, make sure of the project health as per table
below:
.. |beat.cmdline-status| image:: https://gitlab.idiap.ch/ci/projects/5/status.png?ref=master
:target: https://gitlab.idiap.ch/ci/projects/5?ref=master
.. |beat.core-status| image:: https://gitlab.idiap.ch/ci/projects/2/status.png?ref=master
:target: https://gitlab.idiap.ch/ci/projects/2?ref=master
============================= ========================
Package Status (master branch)
============================= ========================
beat.core |beat.core-status|
beat.cmdline (this package) |beat.cmdline-status|
============================= ========================
Installation
------------
Really easy, with ``zc.buildout``::
$ python bootstrap-buildout.py
$ ./bin/buildout
These 2 commands should download and install all non-installed dependencies and
get you a fully operational test and development environment.
.. note::
The python shell used in the first line of the previous command set
determines the python interpreter that will be used for all scripts developed
inside this package.
If you are on the Idiap filesystem, you may use
``/idiap/project/beat/environments/staging/usr/bin/python`` to bootstrap this
package instead. It contains the same setup deployed at the final BEAT
machinery.
Complete BEAT's `installation`_ instructions. Then, to install this package,
run::
$ conda install beat.cmdline
Documentation
-------------
To build the documentation, just do::
$ ./bin/sphinx-apidoc --separate -d 2 --output=doc/api beat beat/cmdline/test beat/cmdline/scripts
$ ./bin/sphinx-build doc sphinx
Testing
Contact
-------
After installation, it is possible to run our suite of unit tests. To do so,
use ``nose``::
$ ./bin/nosetests -sv
.. note::
Some of the tests for our command-line toolkit require a running BEAT
platform web-server, with a compatible ``beat.core`` installed (preferably
the same). By default, these tests will be skipped. If you want to run
them, you must setup a development web server and set the environment
variable ``BEAT_CMDLINE_TEST_PLATFORM`` to point to that address. For
example::
$ export BEAT_CMDLINE_TEST_PLATFORM="http://example.com/platform/"
$ ./bin/nosetests -sv
It is **not** adviseable to run tests against a production web server.
If you want to skip slow tests (at least those pulling stuff from our servers)
or executing lengthy operations, just do::
$ ./bin/nosetests -sv -a '!slow'
To measure the test coverage, do the following::
$ ./bin/nosetests -sv --with-coverage --cover-package=beat.cmdline
To produce an HTML test coverage report, at the directory `./htmlcov`, do the
following::
$ ./bin/nosetests -sv --with-coverage --cover-package=beat.cmdline --cover-html --cover-html-dir=htmlcov
Our documentation is also interspersed with test units. You can run them using
sphinx::
$ ./bin/sphinx -b doctest doc sphinx
Development
-----------
Profiling
=========
In order to profile the test code, try the following::
$ ./bin/python -mcProfile -oprof.data ./bin/nosetests -sv ...
This will dump the profiling data at ``prof.data``. You can dump its contents
in different ways using another command::
For questions or reporting issues to this software package, contact our
development `mailing list`_.
$ ./bin/python -mpstats prof.data
This will allow you to dump and print the profiling statistics as you may find
fit.
.. Place your references here:
.. _beat: https://www.idiap.ch/software/beat
.. _installation: https://www.idiap.ch/software/beat/install
.. _mailing list: https://www.idiap.ch/software/beat/discuss
......@@ -25,5 +25,6 @@
# #
###############################################################################
#see http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
__import__('pkg_resources').declare_namespace(__name__)
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
......@@ -24,7 +24,3 @@
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
#see http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
__import__('pkg_resources').declare_namespace(__name__)
......@@ -25,82 +25,27 @@
# #
###############################################################################
"""Usage:
%(prog)s algorithms list [--remote]
%(prog)s algorithms check [<name>]...
%(prog)s algorithms pull [--force] [<name>]...
%(prog)s algorithms push [--force] [--dry-run] [<name>]...
%(prog)s algorithms diff <name>
%(prog)s algorithms status
%(prog)s algorithms create <name>...
%(prog)s algorithms version <name>
%(prog)s algorithms fork <src> <dst>
%(prog)s algorithms rm [--remote] <name>...
%(prog)s algorithms execute <instructions>
%(prog)s algorithms execute --examples
%(prog)s algorithms --help
Commands:
list Lists all the algorithms available on the platform
check Checks a local algorithm for validity
pull Downloads the specified algorithms from the server
push Uploads algorithms to the server
diff Shows changes between the local algorithm and the remote version
status Shows (editing) status for all available algorithms
create Creates a new local algorithm
version Creates a new version of an existing algorithm
fork Forks a local algorithm
rm Deletes a local algorithm (unless --remote is specified)
execute Execute an algorithm following instructions in a JSON file
Options:
--force Performs operation regardless of conflicts
--dry-run Doesn't really perform the task, just comments what would do
--remote Only acts on the remote copy of the algorithm
--examples Display some example JSON instruction files
--help Display this screen
Arguments for 'execute':
<instructions> JSON file containing the instructions
Examples:
To list all algorithms available locally:
$ %(prog)s al list
To list all algorithms available at the platform:
$ %(prog)s al list --remote
"""
import click
import logging
logger = logging.getLogger(__name__)
import os
import sys
import docopt
import simplejson as json
from . import common
from beat.core import algorithm
from beat.core.execution import DockerExecutor
from beat.core.dock import Host
from beat.core import hash
from beat.backend.python.database import Storage as DatabaseStorage
from beat.backend.python.algorithm import Storage as AlgorithmStorage
from . import common
from .decorators import raise_on_error
from .click_helper import AliasedGroup
logger = logging.getLogger(__name__)
def pull(webapi, prefix, names, force, indentation, format_cache, lib_cache):
def pull_impl(webapi, prefix, names, force, indentation, format_cache, lib_cache):
"""Copies algorithms (and required libraries/dataformats) from the server.
Parameters:
......@@ -111,11 +56,11 @@ def pull(webapi, prefix, names, force, indentation, format_cache, lib_cache):
prefix (str): A string representing the root of the path in which the user
objects are stored
names (list): A list of strings, each representing the unique relative path
of the objects to retrieve or a list of usernames from which to retrieve
objects. If the list is empty, then we pull all available objects of a
given type. If no user is set, then pull all public objects of a given
type.
names (:py:class:`list`): A list of strings, each representing the unique
relative path of the objects to retrieve or a list of usernames from
which to retrieve objects. If the list is empty, then we pull all
available objects of a given type. If no user is set, then pull all
public objects of a given type.
force (bool): If set to ``True``, then overwrites local changes with the
remotely retrieved copies.
......@@ -139,8 +84,8 @@ def pull(webapi, prefix, names, force, indentation, format_cache, lib_cache):
"""
from .dataformats import pull as dataformats_pull
from .libraries import pull as libraries_pull
from .dataformats import pull_impl as dataformats_pull
from .libraries import pull_impl as libraries_pull
status, names = common.pull(webapi, prefix, 'algorithm', names,
['declaration', 'code', 'description'], force, indentation)
......@@ -169,7 +114,7 @@ def pull(webapi, prefix, names, force, indentation, format_cache, lib_cache):
def print_examples():
print """
print("""
To feed data from a database to an algorithm:
=============================================
......@@ -247,11 +192,11 @@ To execute an analyzer:
"version": "<environment_version>"
}
}
"""
""")
def execute(prefix, cache, instructions_file):
def execute_impl(prefix, cache, instructions_file):
try:
# Load the JSON configuration
if not os.path.exists(instructions_file):
......@@ -264,22 +209,21 @@ def execute(prefix, cache, instructions_file):
configuration['queue'] = 'unused'
configuration['nb_slots'] = 1
if not configuration.has_key('parameters'):
if 'parameters' not in configuration:
configuration['parameters'] = {}
for name, cfg in configuration['inputs'].items():
cfg['endpoint'] = name
suffix = ''
if 'database' in cfg: # Connected to a database output
cfg['hash'] = hash.hashDataset(cfg['database'], cfg['protocol'], cfg['set'])
suffix = '.db'
if cfg.has_key('database'): # Connected to a database output
db = DatabaseStorage(prefix, cfg['database'])
cfg['hash'] = hash.hashDatasetOutput(db.hash(), cfg['protocol'], cfg['set'], cfg['output'])
cfg['path'] = hash.toPath(cfg['hash'], '')
else:
cfg['path'] = hash.toPath(cfg['hash'], '')
cfg['path'] = hash.toPath(cfg['hash'], suffix=suffix)
algo = AlgorithmStorage(prefix, configuration['algorithm'])
if configuration.has_key('outputs'): # Standard algorithm
if 'outputs' in configuration: # Standard algorithm
for name, cfg in configuration['outputs'].items():
cfg['endpoint'] = name
cfg['hash'] = hash.hashBlockOutput(
......@@ -323,16 +267,16 @@ def execute(prefix, cache, instructions_file):
with executor:
result = executor.process()
if result['status'] != 0:
print 'STDERR:'
print result['stderr']
print('STDERR:')
print(result['stderr'])
# Display the results
if configuration.has_key('outputs'): # Standard algorithm
print 'Outputs of the algorithms available at:'
if 'outputs' in configuration: # Standard algorithm
print('Outputs of the algorithms available at:')
for name, cfg in configuration['outputs'].items():
print ' - %s: %s' % (name, cfg['path'])
print(' - %s: %s' % (name, cfg['path']))
else:
print 'Results of the analyzer available at: %s' % configuration['result']['path']
print('Results of the analyzer available at: %s' % configuration['result']['path'])
except Exception as e:
import traceback
......@@ -343,64 +287,219 @@ def execute(prefix, cache, instructions_file):
def process(args):
if args['list']:
if args['--remote']:
with common.make_webapi(args['config']) as webapi:
return common.display_remote_list(webapi, 'algorithm')
else:
return common.display_local_list(args['config'].path, 'algorithm')
elif args['check']:
return common.check(args['config'].path, 'algorithm', args['<name>'])
elif args['pull']:
with common.make_webapi(args['config']) as webapi:
return pull(webapi, args['config'].path, args['<name>'],
args['--force'], 0, {}, {})
elif args['push']:
with common.make_webapi(args['config']) as webapi:
return common.push(webapi, args['config'].path, 'algorithm',
args['<name>'], ['name', 'declaration', 'code', 'description'],
{}, args['--force'], args['--dry-run'], 0)
elif args['diff']:
with common.make_webapi(args['config']) as webapi:
return common.diff(webapi, args['config'].path, 'algorithm',
args['<name>'][0], ['declaration', 'code', 'description'])
@click.group(cls=AliasedGroup)
@click.pass_context
def algorithms(ctx):
"""Configuration and manipulation of algorithms"""
pass
@algorithms.command()
@click.option('--remote', help='Only acts on the remote copy of the algorithm',
is_flag=True)
@click.pass_context
@raise_on_error
def list(ctx, remote):
'''Lists all the algorithms available on the platform
Example:
$ beat algorithms list --remote
'''
if remote:
with common.make_webapi(ctx.meta['config']) as webapi:
return common.display_remote_list(webapi, 'algorithm')
else:
return common.display_local_list(ctx.meta['config'].path, 'algorithm')
@algorithms.command()
@click.argument('names', nargs=-1)
@click.pass_context
@raise_on_error
def path(ctx, names):
'''Displays local path of algorithm files
Example:
$ beat algorithms path xxx
'''
return common.display_local_path(ctx.meta['config'].path, 'algorithm', names)
@algorithms.command()
@click.argument('name', nargs=1)
@click.pass_context
@raise_on_error
def edit(ctx, name):
'''Edit local algorithm file
Example:
$ beat algorithms edit xxx
'''
return common.edit_local_file(ctx.meta['config'].path,
ctx.meta['config'].editor, 'algorithm',
name)
@algorithms.command()
@click.argument('name', nargs=-1)
@click.pass_context
@raise_on_error
def check(ctx, name):
<