Commit 2580eac2 authored by André Anjos's avatar André Anjos 💬

Merge branch '1.4.x' into 'master'

1.4.x

Closes #40, #34, and #24

See merge request !54
parents d51714a8 76c8bbbf
Pipeline #25454 passed with stages
in 18 minutes and 57 seconds
......@@ -29,4 +29,8 @@ dataformats/
experiments/
libraries/
toolchains/
plotters/
plotterparameters/
.noseids
scripts/_core_docker_pull.sh
_ci/
# This build file uses template features from YAML so it is generic enough for
# any Bob project. Don't modify it unless you know what you're doing.
# Definition of global variables (all stages)
variables:
CONDA_ROOT: "${CI_PROJECT_DIR}/miniconda"
DOCKER_REGISTRY: docker.idiap.ch
# Definition of our build pipeline order
stages:
- build
- deploy
- pypi
variables:
PREFIX: /opt/beat.env.web/usr
build:
# Build targets
.build_template: &build_job
stage: build
before_script:
- ${PREFIX}/bin/python --version
- docker info
- mkdir _ci
- curl --silent "https://gitlab.idiap.ch/bob/bob.admin/raw/master/gitlab/install.sh" > _ci/install.sh
- chmod 755 _ci/install.sh
- ./_ci/install.sh _ci master #installs ci support scripts
- ./_ci/before_build.sh
- docker info
- ./scripts/docker_pull.sh master #pulls required test images
script:
- ./_ci/build.sh
after_script:
- ./_ci/after_build.sh
cache: &build_caches
paths:
- miniconda.sh
- ${CONDA_ROOT}/pkgs/*.tar.bz2
- ${CONDA_ROOT}/pkgs/urls.txt
.build_linux_template: &linux_build_job
<<: *build_job
tags:
- docker-build
image: continuumio/conda-concourse-ci
artifacts:
expire_in: 1 week
paths:
- _ci/
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
cache:
<<: *build_caches
key: "linux-cache"
.build_macosx_template: &macosx_build_job
<<: *build_job
tags:
- macosx
artifacts:
expire_in: 1 week
paths:
- _ci/
- ${CONDA_ROOT}/conda-bld/osx-64/*.tar.bz2
cache:
<<: *build_caches
key: "macosx-cache"
build_linux_36:
<<: *linux_build_job
variables:
PYTHON_VERSION: "3.6"
BUILD_EGG: "true"
artifacts:
expire_in: 1 week
paths:
- _ci/
- dist/*.zip
- sphinx
- ${CONDA_ROOT}/conda-bld/linux-64/*.tar.bz2
build_macosx_36:
<<: *macosx_build_job
variables:
PYTHON_VERSION: "3.6"
# Deploy targets
.deploy_template: &deploy_job
stage: deploy
before_script:
- ./_ci/install.sh _ci master #updates ci support scripts
script:
- ./_ci/deploy.sh
dependencies:
- build_linux_36
- build_macosx_36
tags:
- deployer
deploy_beta:
<<: *deploy_job
environment: beta
only:
- master
deploy_stable:
<<: *deploy_job
environment: stable
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
pypi:
stage: pypi
environment: pypi
only:
- /^v\d+\.\d+\.\d+([abc]\d*)?$/ # PEP-440 compliant version (tags)
except:
- branches
before_script:
- ./_ci/install.sh _ci master #updates ci support scripts
script:
- git clean -ffdx
- ${PREFIX}/bin/python bootstrap-buildout.py
- ./bin/buildout
- ./bin/python ${PREFIX}/bin/coverage run --source=${CI_PROJECT_NAME} ${PREFIX}/bin/nosetests -sv ${CI_PROJECT_NAME}
- ./bin/python ${PREFIX}/bin/coverage report
- ./bin/python ${PREFIX}/bin/sphinx-apidoc --separate -d 2 --output=doc/api beat
- ./bin/python ${PREFIX}/bin/sphinx-build doc html
- ./_ci/pypi.sh
dependencies:
- build_linux_36
tags:
- docker-build
- deployer
include LICENSE.AGPL README.rst buildout.cfg bootstrap-buildout.py
recursive-include doc conf.py *.rst *.png *.svg *.ico *.pdf
include LICENSE.AGPL README.rst version.txt requirements.txt
include buildout.cfg develop.cfg
recursive-include scripts *.sh
recursive-include doc conf.py *.rst *.png *.ico *.pdf
recursive-include beat *.checksum *.index *.json *.data
......@@ -3,7 +3,7 @@
.. Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ ..
.. Contact: beat.support@idiap.ch ..
.. ..
.. This file is part of the beat.cmdline module of the BEAT platform. ..
.. This file is part of the beat.cmdline module of the BEAT platform. ..
.. ..
.. Commercial License Usage ..
.. Licensees holding valid commercial BEAT licenses may use this file in ..
......@@ -20,123 +20,47 @@
.. You should have received a copy of the GNU Affero Public License along ..
.. with the BEAT platform. If not, see http://www.gnu.org/licenses/. ..
.. image:: https://img.shields.io/badge/docs-stable-yellow.svg
:target: https://www.idiap.ch/software/beat/docs/beat/beat.cmdline/stable/index.html
.. image:: https://img.shields.io/badge/docs-latest-orange.svg
:target: https://www.idiap.ch/software/beat/docs/beat/beat.cmdline/master/index.html
.. image:: https://gitlab.idiap.ch/beat/beat.cmdline/badges/master/build.svg
:target: https://gitlab.idiap.ch/beat/beat.cmdline/commits/master
.. image:: https://gitlab.idiap.ch/beat/beat.cmdline/badges/master/coverage.svg
:target: https://gitlab.idiap.ch/beat/beat.cmdline/commits/master
.. image:: https://img.shields.io/badge/gitlab-project-0000c0.svg
:target: https://gitlab.idiap.ch/beat/beat.cmdline
.. image:: https://img.shields.io/pypi/v/beat.cmdline.svg
:target: https://pypi.python.org/pypi/beat.cmdline
============================================
Biometrics Evaluation and Testing Platform
============================================
This package contains the source code for a python-based command-line client
for the BEAT platform.
==========================
Core Components for BEAT
==========================
Dependence Status
-----------------
This package part of BEAT_, an open-source evaluation platform for data science
algorithms and workflows. It contains the command-line based program that can
download/upload components with the platform. It can also run experiments
locally.
Before checking-out sources, make sure of the project health as per table
below:
.. |beat.cmdline-status| image:: https://gitlab.idiap.ch/ci/projects/5/status.png?ref=master
:target: https://gitlab.idiap.ch/ci/projects/5?ref=master
.. |beat.core-status| image:: https://gitlab.idiap.ch/ci/projects/2/status.png?ref=master
:target: https://gitlab.idiap.ch/ci/projects/2?ref=master
============================= ========================
Package Status (master branch)
============================= ========================
beat.core |beat.core-status|
beat.cmdline (this package) |beat.cmdline-status|
============================= ========================
Installation
------------
Really easy, with ``zc.buildout``::
$ python bootstrap-buildout.py
$ ./bin/buildout
These 2 commands should download and install all non-installed dependencies and
get you a fully operational test and development environment.
.. note::
The python shell used in the first line of the previous command set
determines the python interpreter that will be used for all scripts developed
inside this package.
If you are on the Idiap filesystem, you may use
``/idiap/project/beat/environments/staging/usr/bin/python`` to bootstrap this
package instead. It contains the same setup deployed at the final BEAT
machinery.
Complete BEAT's `installation`_ instructions. Then, to install this package,
run::
$ conda install beat.cmdline
Documentation
-------------
To build the documentation, just do::
$ ./bin/sphinx-apidoc --separate -d 2 --output=doc/api beat beat/cmdline/test beat/cmdline/scripts
$ ./bin/sphinx-build doc sphinx
Testing
Contact
-------
After installation, it is possible to run our suite of unit tests. To do so,
use ``nose``::
$ ./bin/nosetests -sv
.. note::
Some of the tests for our command-line toolkit require a running BEAT
platform web-server, with a compatible ``beat.core`` installed (preferably
the same). By default, these tests will be skipped. If you want to run
them, you must setup a development web server and set the environment
variable ``BEAT_CMDLINE_TEST_PLATFORM`` to point to that address. For
example::
$ export BEAT_CMDLINE_TEST_PLATFORM="http://example.com/platform/"
$ ./bin/nosetests -sv
It is **not** adviseable to run tests against a production web server.
If you want to skip slow tests (at least those pulling stuff from our servers)
or executing lengthy operations, just do::
$ ./bin/nosetests -sv -a '!slow'
To measure the test coverage, do the following::
$ ./bin/nosetests -sv --with-coverage --cover-package=beat.cmdline
To produce an HTML test coverage report, at the directory `./htmlcov`, do the
following::
$ ./bin/nosetests -sv --with-coverage --cover-package=beat.cmdline --cover-html --cover-html-dir=htmlcov
Our documentation is also interspersed with test units. You can run them using
sphinx::
$ ./bin/sphinx -b doctest doc sphinx
Development
-----------
Profiling
=========
In order to profile the test code, try the following::
$ ./bin/python -mcProfile -oprof.data ./bin/nosetests -sv ...
This will dump the profiling data at ``prof.data``. You can dump its contents
in different ways using another command::
For questions or reporting issues to this software package, contact our
development `mailing list`_.
$ ./bin/python -mpstats prof.data
This will allow you to dump and print the profiling statistics as you may find
fit.
.. Place your references here:
.. _beat: https://www.idiap.ch/software/beat
.. _installation: https://www.idiap.ch/software/beat/install
.. _mailing list: https://www.idiap.ch/software/beat/discuss
......@@ -25,5 +25,6 @@
# #
###############################################################################
#see http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
__import__('pkg_resources').declare_namespace(__name__)
# see https://docs.python.org/3/library/pkgutil.html
from pkgutil import extend_path
__path__ = extend_path(__path__, __name__)
......@@ -24,7 +24,3 @@
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
#see http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages
__import__('pkg_resources').declare_namespace(__name__)
This diff is collapsed.
......@@ -25,183 +25,165 @@
# #
###############################################################################
"""Configuration manipulation and display
Usage:
%(prog)s cache clear
%(prog)s cache info [--start=<index>] [--end=<index>] [--sizes] [<path>]...
%(prog)s cache view [--start=<index>] [--end=<index>] [<path>]...
%(prog)s cache --help
Commands:
clear Deletes all available cache
info Displays information about a particular cache file
view Displays the contents of a particular cache file
Options:
-s, --sizes If set, also print the size in bytes for objects in a file. This
triggers the full file readout
--start=<i> If set, allows the user to print only a few bits of the file
--end=<i> If set, allows the user to print only a few bits of the file
-h, --help Display this screen
Examples:
To clear all available cache:
$ %(prog)s cache clear
To collect information about a particular cache file:
$ %(prog)s cache info 7f/d8/8d/a11178ac27075feaba8131fe878d6e3...
To view a particular cache file:
$ %(prog)s cache view 7f/d8/8d/a11178ac27075feaba8131fe878d6e3...
"""
import os
import click
import fnmatch
import logging
logger = logging.getLogger(__name__)
import simplejson
from . import common
from beat.core.data import CachedDataSource, load_data_index
from beat.core.utils import NumpyJSONEncoder
def get_paths(config):
func = lambda z: z.split('.', 1)[0]
retval = []
for dirname, dirs, files in os.walk(config.cache):
files = fnmatch.filter(files, '*.data') #avoid index-only files
if not files: continue
d = dirname.replace(config.cache, '').strip(os.sep)
retval += list(set([os.path.join(d, func(k)) for k in files]))
return retval
def info(config, paths, sizes, index_start, index_end):
if not paths:
paths = get_paths(config)
for path in paths:
logger.info('path: %s', path)
fullpath = os.path.join(config.cache, path + '.data')
f = CachedDataSource()
status = f.setup(fullpath, config.path, index_start, index_end)
if not status:
logger.error("cannot setup data source with `%s' and prefix `%s'",
fullpath, config.path)
return 1
logger.info(' dataformat: %s', f.dataformat.name)
if sizes:
counter = 0
logger.info(' index:')
while f.hasMoreData():
try:
data, start, end = f.next()
except Exception as e:
logger.error("Failed to retrieve the next data: %s", e)
return 1
from . import common
from .decorators import raise_on_error
from .click_helper import AliasedGroup
size = len(data.pack())
counter += size
if start == end:
logger.info(' [%d] - %d bytes', start, size)
else:
logger.info(' [%d:%d] - %d bytes', start, end, size)
logger.info(' total (stripped-down) size: %d bytes', counter)
else:
index = load_data_index(config.cache, path + '.data')
logger.info(' objects : %d', len(index)-1)
logger = logging.getLogger(__name__)
return 0
def get_paths(config):
def view(config, paths, index_start, index_end):
func = lambda z: z.split('.', 1)[0]
if not paths:
paths = get_paths(config)
retval = []
for path in paths:
for dirname, _, files in os.walk(config.cache):
files = fnmatch.filter(files, '*.data') #avoid index-only files
if not files:
continue
d = dirname.replace(config.cache, '').strip(os.sep)
retval += list(set([os.path.join(d, func(k)) for k in files]))
logger.info('path: %s', path)
fullpath = os.path.join(config.cache, path + '.data')
return retval
f = CachedDataSource()
status = f.setup(fullpath, config.path, index_start, index_end)
if not status:
logger.error("cannot setup data source with `%s' and prefix `%s'",
fullpath, config.path)
return 1
logger.info(' dataformat: %s', f.dataformat.name)
@click.group(cls=AliasedGroup)
@click.pass_context
@click.option('--start', type=click.INT, help='If set, allows the user to '
'print only a few bits of the file')
@click.option('--end', type=click.INT, help='If set, allows the user to '
'print only a few bits of the file')
def cache(ctx, start, end):
"""Configuration manipulation and display"""
pass
while f.hasMoreData():
try:
data, start, end = f.next()
except Exception as e:
logger.error("Failed to retrieve the next data: %s", e)
return 1
@cache.command()
@click.pass_context
def clear(ctx):
'''Deletes all available cache
logger.extra(80 * '-')
To clear all available cache:
if start == end:
header = '[%d]: ' % start
else:
header = '[%d:%d]: ' % (start, end)
$ %(prog)s cache clear
'''
import shutil
if os.path.isdir(ctx.meta['config'].cache):
for k in os.listdir(ctx.meta['config'].cache):
p = os.path.join(ctx.meta['config'].cache, k)
shutil.rmtree(p)
json_data = data.as_dict()
for name, value in json_data.items():
json_data[name] = common.stringify(value)
json_data = simplejson.dumps(json_data, indent=2,
cls=NumpyJSONEncoder).\
replace('"BEAT_LIST_DELIMITER[', '[')\
.replace(']BEAT_LIST_DELIMITER"', ']')\
.replace('"...",', '...')\
.replace('"BEAT_LIST_SIZE(', '(')\
.replace(')BEAT_LIST_SIZE"', ')')
logger.info(header + json_data)
return 0
@cache.command()
@click.argument('paths', nargs=-1, type=click.Path(exists=True))
@click.pass_context
@click.option('--sizes', help='If set, also print the size in bytes for '
'objects in a file. This triggers the full file readout',
is_flag=True)
def info(ctx, paths, sizes):
'''Displays information about a particular cache file
To collect information about a particular cache file:
def process(args):
$ %(prog)s cache info 7f/d8/8d/a11178ac27075feaba8131fe878d6e3...
'''
config = ctx.meta['config']
index_start = int(ctx.meta['start']) if 'start' in ctx.meta else None
index_end = int(ctx.meta['end']) if 'end' in ctx.meta else None
if not paths:
paths = get_paths(config)
for path in paths:
logger.info('path: %s', path)
fullpath = os.path.join(config.cache, path + '.data')
f = CachedDataSource()
status = f.setup(fullpath, config.path, index_start, index_end)
if not status:
logger.error("cannot setup data source with `%s' and prefix `%s'",
fullpath, config.path)
return 1
logger.info(' dataformat: %s', f.dataformat.name)
if sizes:
counter = 0
logger.info(' index:')
for data, start, end in f:
size = len(data.pack())
counter += size
if start == end:
logger.info(' [%d] - %d bytes', start, size)
else:
logger.info(' [%d:%d] - %d bytes', start, end, size)
logger.info(' total (stripped-down) size: %d bytes', counter)
start = int(args['--start']) if args['--start'] is not None else None
end = int(args['--end']) if args['--end'] is not None else None
else:
index = load_data_index(config.cache, path + '.data')
logger.info(' objects : %d', len(index)-1)
if args['clear']:
import shutil
for k in os.listdir(args['config'].cache):
p = os.path.join(args['config'].cache, k)
shutil.rmtree(p)
return 0
elif args['info']:
return info(args['config'], args['<path>'], args['--sizes'], start, end)
@cache.command()
@click.argument('paths', nargs=-1)
@click.pass_context
@raise_on_error
def view(ctx, paths):
'''Displays information about a particular cache file
elif args['view']:
return view(args['config'], args['<path>'], start, end)
To view a particular cache file:
# Should not happen
logger.error("unrecognized `cache' subcommand")
return 1
$ %(prog)s cache view 7f/d8/8d/a11178ac27075feaba8131fe878d6e3...
'''
config = ctx.meta['config']
index_start = int(ctx.meta['start']) if 'start' in ctx.meta else None
index_end = int(ctx.meta['end']) if 'end' in ctx.meta else None
if not paths:
paths = get_paths(config)
for path in paths:
logger.info('path: %s', path)
fullpath = os.path.join(config.cache, path + '.data')
f = CachedDataSource()
status = f.setup(fullpath, config.path, index_start, index_end)
if not status:
logger.error("cannot setup data source with `%s' and prefix `%s'",
fullpath, config.path)
return 1
logger.info(' dataformat: %s', f.dataformat.name)
for data, start, end in f:
logger.extra(80 * '-')
if start == end:
header = '[%d]: ' % start
else:
header = '[%d:%d]: ' % (start, end)
json_data = data.as_dict()
for name, value in json_data.items():
json_data[name] = common.stringify(value)
json_data = simplejson.dumps(
json_data, indent=2,
cls=NumpyJSONEncoder).\
replace('"BEAT_LIST_DELIMITER[', '[')\
.replace(']BEAT_LIST_DELIMITER"', ']')\
.replace('"...",', '...')\
.replace('"BEAT_LIST_SIZE(', '(')\
.replace(')BEAT_LIST_SIZE"', ')')
logger.info(header + json_data)
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.cmdline module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
import click
class AliasedGroup(click.Group):
''' Class that handles prefix aliasing for commands '''
def get_command(self, ctx, cmd_name):
rv = click.Group.get_command(self, ctx, cmd_name)
if rv is not None:
return rv