Commit cf132484 authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME
Browse files

Merge branch 'fix_click_commands' into '1.4.x'

Fix click commands

See merge request !41
parents eb1e771e af5ae886
Pipeline #21666 passed with stages
in 21 minutes and 1 second
......@@ -28,20 +28,19 @@
import click
import logging
import os
import sys
import docopt
import simplejson as json
from . import common
from beat.cmdline.scripts.click_helper import AliasedGroup
from beat.core import algorithm
from beat.core.execution import DockerExecutor
from beat.core.dock import Host
from beat.core import hash
from beat.backend.python.database import Storage as DatabaseStorage
from beat.backend.python.algorithm import Storage as AlgorithmStorage
from . import common
from .decorators import raise_on_error
from .click_helper import AliasedGroup
logger = logging.getLogger(__name__)
......@@ -298,6 +297,7 @@ def algorithms(ctx):
@click.option('--remote', help='Only acts on the remote copy of the algorithm',
is_flag=True)
@click.pass_context
@raise_on_error
def list(ctx, remote):
'''Lists all the algorithms available on the platform
......@@ -313,6 +313,7 @@ def list(ctx, remote):
@algorithms.command()
@click.argument('names', nargs=-1)
@click.pass_context
@raise_on_error
def path(ctx, names):
'''Displays local path of algorithm files
......@@ -325,6 +326,7 @@ def path(ctx, names):
@algorithms.command()
@click.argument('name', nargs=1)
@click.pass_context
@raise_on_error
def edit(ctx, name):
'''Edit local algorithm file
......@@ -339,6 +341,7 @@ def edit(ctx, name):
@algorithms.command()
@click.argument('name', nargs=-1)
@click.pass_context
@raise_on_error
def check(ctx, name):
'''Checks a local algorithm for validity
......@@ -354,6 +357,7 @@ def check(ctx, name):
@click.option('--force', help='Performs operation regardless of conflicts',
is_flag=True)
@click.pass_context
@raise_on_error
def pull(ctx, name, force):
'''Downloads the specified algorithms from the server
......@@ -373,6 +377,7 @@ def pull(ctx, name, force):
@click.option('--dry-run', help="Doesn't really perform the task, just "
"comments what would do", is_flag=True)
@click.pass_context
@raise_on_error
def push(ctx, name, force, dry_run):
'''Uploads algorithms to the server
......@@ -389,6 +394,7 @@ def push(ctx, name, force, dry_run):
@algorithms.command()
@click.argument('name', nargs=1)
@click.pass_context
@raise_on_error
def diff(ctx, name):
'''Shows changes between the local algorithm and the remote version
......@@ -403,6 +409,7 @@ def diff(ctx, name):
@algorithms.command()
@click.pass_context
@raise_on_error
def status(ctx):
'''Shows (editing) status for all available algorithms
......@@ -417,6 +424,7 @@ def status(ctx):
@algorithms.command()
@click.argument('name', nargs=-1)
@click.pass_context
@raise_on_error
def create(ctx, name):
'''Creates a new local algorithm
......@@ -430,6 +438,7 @@ def create(ctx, name):
@algorithms.command()
@click.argument('name', nargs=1)
@click.pass_context
@raise_on_error
def version(ctx, name):
'''Creates a new version of an existing algorithm
......@@ -444,6 +453,7 @@ def version(ctx, name):
@click.argument('src', nargs=1)
@click.argument('dst', nargs=1)
@click.pass_context
@raise_on_error
def fork(ctx, src, dst):
'''Forks a local algorithm
......@@ -459,6 +469,7 @@ def fork(ctx, src, dst):
@click.option('--remote', help='Only acts on the remote copy of the algorithm',
is_flag=True)
@click.pass_context
@raise_on_error
def rm(ctx, name, remote):
'''Deletes a local algorithm (unless --remote is specified)
......@@ -478,6 +489,7 @@ def rm(ctx, name, remote):
@click.option('--example', help='Display some example JSON instruction files',
is_flag=True)
@click.pass_context
@raise_on_error
def execute(ctx, instructions, example):
'''Execute an algorithm following instructions in a JSON file
......
......@@ -29,15 +29,20 @@ import os
import click
import fnmatch
import logging
logger = logging.getLogger(__name__)
import simplejson
from . import common
from beat.cmdline.scripts.click_helper import AliasedGroup
from beat.core.data import CachedDataSource, load_data_index
from beat.core.utils import NumpyJSONEncoder
from . import common
from .decorators import raise_on_error
from .click_helper import AliasedGroup
logger = logging.getLogger(__name__)
def get_paths(config):
func = lambda z: z.split('.', 1)[0]
......@@ -135,6 +140,7 @@ def info(ctx, paths, sizes):
@cache.command()
@click.argument('paths', nargs=-1)
@click.pass_context
@raise_on_error
def view(ctx, paths):
'''Displays information about a particular cache file
......
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.cmdline module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
import click
class AliasedGroup(click.Group):
''' Class that handles prefix aliasing for commands '''
def get_command(self, ctx, cmd_name):
rv = click.Group.get_command(self, ctx, cmd_name)
if rv is not None:
return rv
matches = [x for x in self.list_commands(ctx)
if x.startswith(cmd_name)]
if not matches:
return None
elif len(matches) == 1:
return click.Group.get_command(self, ctx, matches[0])
ctx.fail('Too many matches: %s' % ', '.join(sorted(matches)))
......@@ -725,9 +725,7 @@ def check(prefix, type, names):
"""
names = make_up_local_list(prefix, type, names)
n = sum([check_one(prefix, type, name) for name in names])
sys.exit(n)
return n
return sum([check_one(prefix, type, name) for name in names])
def fetch_object(webapi, type, name, fields):
......@@ -1172,7 +1170,6 @@ def delete_local(prefix, type, names):
storage.remove()
sys.exit(status)
return status
......@@ -1218,7 +1215,7 @@ def delete_remote(webapi, type, names):
"reason: %s", type, webapi.platform, webapi.user,
six.moves.http_client.responses[status_code])
status += 1
sys.exit(status)
return status
......@@ -1295,7 +1292,7 @@ def status(webapi, prefix, type):
for key in set(remote.keys()) - set(local):
logger.extra('[r] %s/%s (@%s)', TYPE_PLURAL[type], key,
remote[key]['creation_date'])
sys.exit(0)
return 0, retval
......
......@@ -33,9 +33,12 @@ import copy
import logging
import getpass
import click
from .scripts.click_helper import verbosity_option, AliasedGroup
import simplejson
from .decorators import verbosity_option
from .click_helper import AliasedGroup
logger = logging.getLogger(__name__)
......
......@@ -26,31 +26,30 @@
###############################################################################
import os
import sys
import click
import glob
import random
import zmq
import logging
import simplejson
from beat.cmdline.scripts.click_helper import AliasedGroup
from beat.core.hash import toPath
from beat.core.hash import hashDataset
from beat.core.utils import NumpyJSONEncoder
from beat.core.database import Database
from beat.core.data import load_data_index, RemoteDataSource
from beat.core.data import RemoteDataSource
from beat.core import dock
from beat.core import inputs
from beat.core import utils
from . import common
from .decorators import raise_on_error
from .click_helper import AliasedGroup
logger = logging.getLogger(__name__)
CMD_DB_INDEX = 'index'
CMD_VIEW_OUTPUTS = 'databases_provider'
......@@ -466,7 +465,6 @@ def view_outputs(configuration, dataset_name, excluded_outputs=None, uid=None,
# Load the infos about the database set
(db_name, database, sets) = load_database_sets(configuration, dataset_name)
if (database is None) or (len(sets) != 1):
sys.exit(1)
return 1
(protocol_name, set_name, db_set) = sets[0]
......@@ -480,7 +478,6 @@ def view_outputs(configuration, dataset_name, excluded_outputs=None, uid=None,
view = database.view(protocol_name, set_name)
if view is None:
sys.exit(1)
return 1
dataset_hash = hashDataset(db_name, protocol_name, set_name)
......@@ -560,10 +557,8 @@ def view_outputs(configuration, dataset_name, excluded_outputs=None, uid=None,
except Exception as e:
logger.error("Failed to retrieve the next data: %s", e)
sys.exit(1)
return 1
sys.exit(0)
return 0
......@@ -581,6 +576,7 @@ def databases(ctx):
@click.option('--remote', help='Only acts on the remote copy of the database.',
is_flag=True)
@click.pass_context
@raise_on_error
def list(ctx, remote):
'''Lists all the databases available on the platform.
......@@ -599,6 +595,7 @@ def list(ctx, remote):
@databases.command()
@click.argument('names', nargs=-1)
@click.pass_context
@raise_on_error
def path(ctx, names):
'''Displays local path of databases files
......@@ -611,6 +608,7 @@ def path(ctx, names):
@databases.command()
@click.argument('name', nargs=1)
@click.pass_context
@raise_on_error
def edit(ctx, name):
'''Edit local database file
......@@ -626,6 +624,7 @@ def edit(ctx, name):
@databases.command()
@click.argument('db_names', nargs=-1)
@click.pass_context
@raise_on_error
def check(ctx, db_names):
'''Checks a local database for validity.
......@@ -642,6 +641,7 @@ def check(ctx, db_names):
@click.option('--force', help='Performs operation regardless of conflicts',
is_flag=True)
@click.pass_context
@raise_on_error
def pull(ctx, db_names, force):
'''Downloads the specified databases from the server.
......@@ -662,6 +662,7 @@ def pull(ctx, db_names, force):
@click.option('--dry-run', help='Dry run',
is_flag=True)
@click.pass_context
@raise_on_error
def push(ctx, db_names, force, dry_run):
'''Uploads databases to the server (must provide a valid admin token).
......@@ -682,6 +683,7 @@ def push(ctx, db_names, force, dry_run):
@databases.command()
@click.argument('db_names', nargs=-1)
@click.pass_context
@raise_on_error
def diff(ctx, db_names):
'''Shows changes between the local database and the remote version.
......@@ -701,6 +703,7 @@ def diff(ctx, db_names):
@databases.command()
@click.pass_context
@raise_on_error
def status(ctx):
'''Shows (editing) status for all available databases'''
configuration = ctx.meta['config']
......@@ -711,6 +714,7 @@ def status(ctx):
@databases.command()
@click.argument('db_names', nargs=-1)
@click.pass_context
@raise_on_error
def version(ctx, db_names):
'''Creates a new version of an existing database.
......@@ -739,6 +743,7 @@ def version(ctx, db_names):
@click.option('--db-root', help="Database root")
@click.option('--docker', is_flag=True)
@click.pass_context
@raise_on_error
def index(ctx, db_names, list, delete, checksum, uid, db_root, docker):
'''Indexes all outputs (of all sets) of a database.
......@@ -763,7 +768,6 @@ def index(ctx, db_names, list, delete, checksum, uid, db_root, docker):
elif checksum:
code = index_outputs(configuration, db_names, uid=uid,
db_root=db_root, docker=docker)
sys.exit(code)
return code
@databases.command()
......@@ -774,6 +778,7 @@ def index(ctx, db_names, list, delete, checksum, uid, db_root, docker):
@click.option('--db-root', help="Database root")
@click.option('--docker', is_flag=True)
@click.pass_context
@raise_on_error
def view(ctx, set_name, exclude, uid, db_root, docker):
'''View the data of the specified dataset.
......
......@@ -30,12 +30,15 @@ import click
import oset
from beat.cmdline.scripts.click_helper import AliasedGroup
from beat.core import dataformat
from . import common
from .decorators import raise_on_error
from .click_helper import AliasedGroup
from beat.core import dataformat
logger = logging.getLogger(__name__)
LOGGER = logging.getLogger(__name__)
def pull_impl(webapi, prefix, names, force, indentation, cache):
"""Copies dataformats (recursively) from the server.
......@@ -101,7 +104,7 @@ def pull_impl(webapi, prefix, names, force, indentation, cache):
dataformats |= obj.referenced.keys()
except Exception as e:
LOGGER.error("loading `%s': %s...", name, str(e))
logger.error("loading `%s': %s...", name, str(e))
cache[name] = None
# recurse until done
......@@ -118,6 +121,7 @@ def dataformats(ctx):
@click.option('--remote', help='Only acts on the remote copy of the dataformat',
is_flag=True)
@click.pass_context
@raise_on_error
def list(ctx, remote):
'''Lists all the dataformats available on the platform
......@@ -134,6 +138,7 @@ def list(ctx, remote):
@dataformats.command()
@click.argument('names', nargs=-1)
@click.pass_context
@raise_on_error
def path(ctx, names):
'''Displays local path of dataformats files
......@@ -146,6 +151,7 @@ def path(ctx, names):
@dataformats.command()
@click.argument('name', nargs=1)
@click.pass_context
@raise_on_error
def edit(ctx, name):
'''Edit local dataformat file
......@@ -160,6 +166,7 @@ def edit(ctx, name):
@dataformats.command()
@click.argument('names', nargs=-1)
@click.pass_context
@raise_on_error
def check(ctx, names):
'''Checks a local dataformat for validity
......@@ -174,6 +181,7 @@ def check(ctx, names):
@click.option('--force', help='Performs operation regardless of conflicts',
is_flag=True)
@click.pass_context
@raise_on_error
def pull(ctx, name, force):
'''Downloads the specified dataformats from the server
......@@ -194,6 +202,7 @@ def pull(ctx, name, force):
@click.option('--dry-run', help="Doesn't really perform the task, just "
"comments what would do", is_flag=True)
@click.pass_context
@raise_on_error
def push(ctx, name, force, dry_run):
'''Uploads dataformats to the server
......@@ -211,6 +220,7 @@ def push(ctx, name, force, dry_run):
@dataformats.command()
@click.argument('name', nargs=1)
@click.pass_context
@raise_on_error
def diff(ctx, name):
'''Shows changes between the local dataformat and the remote version
......@@ -224,6 +234,7 @@ def diff(ctx, name):
@dataformats.command()
@click.pass_context
@raise_on_error
def status(ctx):
'''Shows (editing) status for all available dataformats
......@@ -237,6 +248,7 @@ def status(ctx):
@dataformats.command()
@click.argument('names', nargs=-1)
@click.pass_context
@raise_on_error
def create(ctx, names):
'''Creates a new local dataformat
......@@ -249,6 +261,7 @@ def create(ctx, names):
@dataformats.command()
@click.argument('name', nargs=1)
@click.pass_context
@raise_on_error
def version(ctx, name):
'''Creates a new version of an existing dataformat
......@@ -262,6 +275,7 @@ def version(ctx, name):
@click.argument('src', nargs=1)
@click.argument('dst', nargs=1)
@click.pass_context
@raise_on_error
def fork(ctx, src, dst):
'''Forks a local dataformat
......@@ -276,6 +290,7 @@ def fork(ctx, src, dst):
@click.option('--remote', help='Only acts on the remote copy of the dataformat',
is_flag=True)
@click.pass_context
@raise_on_error
def rm(ctx, name, remote):
'''Deletes a local dataformat (unless --remote is specified)
......
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.cmdline module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
import logging
import click
from functools import wraps
from .log import set_verbosity_level
# This needs to be beat so that logger is configured for all beat packages.
logger = logging.getLogger('beat')
def verbosity_option(**kwargs):
"""Adds a -v/--verbose option to a click command.
Parameters
----------
**kwargs
All kwargs are passed to click.option.
Returns
-------
callable
A decorator to be used for adding this option.
"""
def custom_verbosity_option(f):
def callback(ctx, param, value):
ctx.meta['verbosity'] = value
set_verbosity_level(logger, value)
logger.debug("Logging of the `beat' logger was set to %d", value)
return value
return click.option(
'-v', '--verbose', count=True,
expose_value=False, default=2,
help="Increase the verbosity level from 0 (only error messages) "
"to 1 (warnings), 2 (log messages), 3 (debug information) by "
"adding the --verbose option as often as desired "
"(e.g. '-vvv' for debug).",
callback=callback, **kwargs)(f)
return custom_verbosity_option
def raise_on_error(view_func):
"""Raise a click exception if returned value is not zero.
Click exits successfully if anything is returned, in order to exit properly
when something went wrong an exception must be raised.
"""
def _decorator(*args, **kwargs):
value = view_func(*args, **kwargs)
if value not in [None, 0]:
exception = click.ClickException("Error occured")
exception.exit_code = value
raise exception
return value
return wraps(view_func)(_decorator)
......@@ -27,16 +27,12 @@
import os
import sys
import logging
import glob
import click
import oset
import simplejson
from . import common
from beat.cmdline.scripts.click_helper import AliasedGroup
from beat.core.experiment import Experiment
from beat.core.execution import DockerExecutor
from beat.core.execution import LocalExecutor
......@@ -46,8 +42,12 @@ from beat.core.dock import Host
from beat.core.hash import toPath
from beat.core.hash import hashDataset
from . import common
from .plotters import plot_impl as plotters_plot
from .plotters import pull_impl as plotters_pull
from .decorators import raise_on_error
from .click_helper import AliasedGroup
logger = logging.getLogger(__name__)
......@@ -144,7 +144,6 @@ def run_experiment(configuration, name, force, use_docker, use_local):