Commit 774429d5 authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME

[cache,databases,dataformats,experiments,plotterparameters,plotters,toolchains]

Changes command inputs

Some arguments are lists instead of strings.
parent f58a358c
......@@ -80,7 +80,7 @@ def clear(ctx):
if os.path.isfile(p):
shutil.rmtree(p)
else:
raise click.ClickException("Failed to clear cache %s" % p)
raise click.ClickException("Failed to clear cache %s." % p)
else:
raise click.ClickException("Cache directory does not exist: %s" %
ctx.meta['config'].cache)
......
......@@ -26,6 +26,7 @@
###############################################################################
import os
import sys
import click
import glob
import random
......@@ -302,7 +303,7 @@ def pull_impl(webapi, prefix, names, force, indentation, format_cache):
"""
from .dataformats import pull as dataformats_pull
from .dataformats import pull_impl as dataformats_pull
status, names = common.pull(webapi, prefix, 'database', names,
['declaration', 'code', 'description'],
......@@ -464,6 +465,7 @@ def view_outputs(configuration, dataset_name, excluded_outputs=None, uid=None,
# Load the infos about the database set
(db_name, database, sets) = load_database_sets(configuration, dataset_name)
if (database is None) or (len(sets) != 1):
sys.exit(1)
return 1
(protocol_name, set_name, db_set) = sets[0]
......@@ -477,6 +479,7 @@ def view_outputs(configuration, dataset_name, excluded_outputs=None, uid=None,
view = database.view(protocol_name, set_name)
if view is None:
sys.exit(1)
return 1
dataset_hash = hashDataset(db_name, protocol_name, set_name)
......@@ -556,8 +559,10 @@ def view_outputs(configuration, dataset_name, excluded_outputs=None, uid=None,
except Exception as e:
logger.error("Failed to retrieve the next data: %s", e)
sys.exit(1)
return 1
sys.exit(0)
return 0
......@@ -566,12 +571,12 @@ def view_outputs(configuration, dataset_name, excluded_outputs=None, uid=None,
@click.group()
@click.pass_context
def db(ctx):
def databases(ctx):
"""Database commands"""
pass
@db.command()
@databases.command()
@click.option('--remote', help='Only acts on the remote copy of the database.',
is_flag=True)
@click.pass_context
......@@ -580,7 +585,7 @@ def list(ctx, remote):
To list all existing databases on your local prefix:
$ beat db list
$ beat databases list
'''
configuration = ctx.meta['config']
if remote:
......@@ -590,13 +595,13 @@ def list(ctx, remote):
return common.display_local_list(configuration.path, 'database')
@db.command()
@databases.command()
@click.argument('db_names', nargs=-1)
@click.pass_context
def check(ctx, db_names):
'''Checks a local database for validity.
$ beat db check [<name>]...
$ beat databases check [<name>]...
<name>:
Database name formatted as "<database>/<version>"
......@@ -604,7 +609,7 @@ def check(ctx, db_names):
return common.check(ctx.meta['config'].path, 'database', db_names)
@db.command()
@databases.command()
@click.argument('db_names', nargs=-1)
@click.option('--force', help='Performs operation regardless of conflicts',
is_flag=True)
......@@ -612,7 +617,7 @@ def check(ctx, db_names):
def pull(ctx, db_names, force):
'''Downloads the specified databases from the server.
$ beat db pull [<name>]...
$ beat databases pull [<name>]...
<name>:
Database name formatted as "<database>/<version>"
......@@ -622,7 +627,7 @@ def pull(ctx, db_names, force):
return pull_impl(webapi, configuration.path, db_names, force, 0, {})
@db.command()
@databases.command()
@click.argument('db_names', nargs=-1)
@click.option('--force', help='Performs operation regardless of conflicts',
is_flag=True)
......@@ -632,7 +637,7 @@ def pull(ctx, db_names, force):
def push(ctx, db_names, force, dry_run):
'''Uploads databases to the server (must provide a valid admin token).
$ beat db push [<name>]...
$ beat databases push [<name>]...
<name>:
Database name formatted as "<database>/<version>"
......@@ -646,13 +651,13 @@ def push(ctx, db_names, force, dry_run):
{}, force, dry_run, 0)
@db.command()
@databases.command()
@click.argument('db_names', nargs=-1)
@click.pass_context
def diff(ctx, db_names):
'''Shows changes between the local database and the remote version.
$ beat db diff [<name>]...
$ beat databases diff [<name>]...
<name>:
Database name formatted as "<database>/<version>"
......@@ -666,7 +671,7 @@ def diff(ctx, db_names):
['declaration', 'code', 'description'])
@db.command()
@databases.command()
@click.pass_context
def status(ctx):
'''Shows (editing) status for all available databases'''
......@@ -675,13 +680,13 @@ def status(ctx):
return common.status(webapi, configuration.path, 'database')[0]
@db.command()
@databases.command()
@click.argument('db_names', nargs=-1)
@click.pass_context
def version(ctx, db_names):
'''Creates a new version of an existing database.
$ beat db version [<name>]...
$ beat databases version [<name>]...
<name>:
Database name formatted as "<database>/<version>"
......@@ -693,14 +698,15 @@ def version(ctx, db_names):
return common.new_version(configuration.path, 'database', db_names[0])
@db.command()
@databases.command()
@click.argument('db_names', nargs=-1)
@click.option('--list', help='List index files matching output if they exist',
is_flag=True)
@click.option('--delete', help='Delete index files matching output if they '
'exist (also, recursively deletes empty directories)',
is_flag=True)
@click.option('--checksum', help='Checksums index files', is_flag=True)
@click.option('--checksum', help='Checksums index files', is_flag=True,
default=True)
@click.option('--uid', type=click.INT, default=None)
@click.option('--db-root', help="Database root")
@click.option('--docker', is_flag=True)
......@@ -710,30 +716,30 @@ def index(ctx, db_names, list, delete, checksum, uid, db_root, docker):
To index the contents of a database
$ beat db index simple/1
$ beat databases index simple/1
To index the contents of a protocol on a database
$ beat db index simple/1/double
$ beat databases index simple/1/double
To index the contents of a set in a protocol on a database
$ beat db index simple/1/double/double
$ beat databases index simple/1/double/double
'''
configuration = ctx.meta['config']
code = 1
if list:
return list_index_files(configuration, db_names)
code = list_index_files(configuration, db_names)
elif delete:
return delete_index_files(configuration, db_names)
code = delete_index_files(configuration, db_names)
elif checksum:
return index_outputs(
configuration, db_names, uid=uid,
db_root=db_root, docker=docker
)
code = index_outputs(configuration, db_names, uid=uid,
db_root=db_root, docker=docker)
sys.exit(code)
return code
@db.command()
@click.argument('db_names', nargs=1)
@databases.command()
@click.argument('set_name', nargs=1)
@click.option('--exclude', help='When viewing, excludes this output',
default=None)
@click.option('--uid', type=click.INT, default=None)
......@@ -745,7 +751,7 @@ def view(ctx, set_name, exclude, uid, db_root, docker):
To view the contents of a specific set
$ beat db view simple/1/protocol/set
$ beat databases view simple/1/protocol/set
'''
configuration = ctx.meta['config']
if exclude is not None:
......
......@@ -108,12 +108,12 @@ def pull_impl(webapi, prefix, names, force, indentation, cache):
@click.group()
@click.pass_context
def df(ctx):
def dataformats(ctx):
"""Configuration manipulation of data formats"""
pass
@df.command()
@dataformats.command()
@click.option('--remote', help='Only acts on the remote copy of the dataformat',
is_flag=True)
@click.pass_context
......@@ -121,7 +121,7 @@ def list(ctx, remote):
'''Lists all the dataformats available on the platform
Example:
$ beat df list --remote
$ beat dataformats list --remote
'''
if remote:
with common.make_webapi(ctx.meta['config']) as webapi:
......@@ -130,7 +130,7 @@ def list(ctx, remote):
return common.display_local_list(ctx.meta['config'].path, 'dataformat')
@df.command()
@dataformats.command()
@click.argument('names', nargs=-1)
@click.pass_context
def path(ctx, names):
......@@ -142,7 +142,7 @@ def path(ctx, names):
return common.display_local_path(ctx.meta['config'].path, 'dataformat', names)
@df.command()
@dataformats.command()
@click.argument('name', nargs=1)
@click.pass_context
def edit(ctx, name):
......@@ -156,20 +156,20 @@ def edit(ctx, name):
name)
@df.command()
@dataformats.command()
@click.argument('names', nargs=-1)
@click.pass_context
def check(ctx, names):
'''Checks a local dataformat for validity
Example:
$ beat df check xxx
$ beat dataformats check xxx
'''
return common.check(ctx.meta['config'].path, 'dataformat', names)
@df.command()
@click.argument('name', nargs=1)
@dataformats.command()
@click.argument('name', nargs=-1)
@click.option('--force', help='Performs operation regardless of conflicts',
is_flag=True)
@click.pass_context
......@@ -177,7 +177,7 @@ def pull(ctx, name, force):
'''Downloads the specified dataformats from the server
Example:
$ beat df pull --force yyy
$ beat dataformats pull --force yyy
'''
with common.make_webapi(ctx.meta['config']) as webapi:
name = common.make_up_remote_list(webapi, 'dataformat', name)
......@@ -186,8 +186,8 @@ def pull(ctx, name, force):
return pull_impl(webapi, ctx.meta['config'].path, name,force, 0, {})
@df.command()
@click.argument('name', nargs=1)
@dataformats.command()
@click.argument('name', nargs=-1)
@click.option('--force', help='Performs operation regardless of conflicts',
is_flag=True)
@click.option('--dry-run', help="Doesn't really perform the task, just "
......@@ -197,7 +197,7 @@ def push(ctx, name, force, dry_run):
'''Uploads dataformats to the server
Example:
$ beat df push --dry-run yyy
$ beat dataformats push --dry-run yyy
'''
with common.make_webapi(ctx.meta['config']) as webapi:
return common.push(
......@@ -207,57 +207,57 @@ def push(ctx, name, force, dry_run):
)
@df.command()
@dataformats.command()
@click.argument('name', nargs=1)
@click.pass_context
def diff(ctx, name):
'''Shows changes between the local dataformat and the remote version
Example:
$ beat df diff xxx
$ beat dataformats diff xxx
'''
with common.make_webapi(ctx.meta.get('config')) as webapi:
return common.diff(webapi, ctx.meta.get('config').path, 'dataformat',
name, ['declaration', 'description'])
@df.command()
@dataformats.command()
@click.pass_context
def status(ctx):
'''Shows (editing) status for all available dataformats
Example:
$ beat df status
$ beat dataformats status
'''
with common.make_webapi(ctx.meta['config']) as webapi:
return common.status(webapi, ctx.meta['config'].path, 'dataformat')[0]
@df.command()
@dataformats.command()
@click.argument('names', nargs=-1)
@click.pass_context
def create(ctx, names):
'''Creates a new local dataformat
Example:
$ beat df create xxx
$ beat dataformats create xxx
'''
return common.create(ctx.meta['config'].path, 'dataformat', names)
@df.command()
@dataformats.command()
@click.argument('name', nargs=1)
@click.pass_context
def version(ctx, name):
'''Creates a new version of an existing dataformat
Example:
$ beat df version xxx
$ beat dataformats version xxx
'''
return common.new_version(ctx.meta['config'].path, 'dataformat', name)
@df.command()
@dataformats.command()
@click.argument('src', nargs=1)
@click.argument('dst', nargs=1)
@click.pass_context
......@@ -265,13 +265,13 @@ def fork(ctx, src, dst):
'''Forks a local dataformat
Example:
$ beat df fork xxx yyy
$ beat dataformats fork xxx yyy
'''
return common.fork(ctx.meta['config'].path, 'dataformat', src, dst)
@df.command()
@click.argument('name', nargs=1)
@dataformats.command()
@click.argument('name', nargs=-1)
@click.option('--remote', help='Only acts on the remote copy of the dataformat',
is_flag=True)
@click.pass_context
......@@ -279,7 +279,7 @@ def rm(ctx, name, remote):
'''Deletes a local dataformat (unless --remote is specified)
Example:
$ beat df rm xxx
$ beat dataformats rm xxx
'''
if remote:
with common.make_webapi(ctx.meta['config']) as webapi:
......
......@@ -27,6 +27,7 @@
import os
import sys
import logging
import glob
import click
......@@ -142,6 +143,7 @@ def run_experiment(configuration, name, force, use_docker, use_local):
logger.error("Failed to load the experiment `%s':", name)
for e in experiment.errors:
logger.error(' * %s', e)
sys.exit(1)
return 1
if not os.path.exists(configuration.cache):
......@@ -169,6 +171,7 @@ def run_experiment(configuration, name, force, use_docker, use_local):
logger.error("Cannot execute block `%s' on environment `%s': "
"environment was not found' - please install it",
key, search_key)
sys.exit(1)
return 1
if use_docker:
......@@ -190,6 +193,7 @@ def run_experiment(configuration, name, force, use_docker, use_local):
"Failed to load the execution information for `%s':", key)
for e in executor.errors:
logger.error(' * %s', e)
sys.exit(1)
return 1
if executor.outputs_exist and not force:
......@@ -221,6 +225,7 @@ def run_experiment(configuration, name, force, use_docker, use_local):
logger.error(" Captured system error:\n%s",
reindent(result['system_error'], 4))
logger.extra(" Environment: %s" % 'default environment')
sys.exit(1)
return 1
elif use_docker:
stats = result['statistics']
......@@ -259,6 +264,7 @@ def run_experiment(configuration, name, force, use_docker, use_local):
for name, details in executor.data['outputs'].items():
logger.extra(" * %s", details['path'])
sys.exit(0)
return 0
......@@ -362,8 +368,8 @@ def pull_impl(webapi, prefix, names, force, indentation, format_cache):
"""
from .algorithms import pull as algorithms_pull
from .databases import pull as databases_pull
from .algorithms import pull_impl as algorithms_pull
from .databases import pull_impl as databases_pull
status, names = common.pull(webapi, prefix, 'experiment', names,
['declaration', 'description'], force,
......@@ -569,7 +575,7 @@ def run(ctx, name, force, docker, local):
def caches(ctx, name, list, delete, checksum):
'''Lists all cache files used by this experiment'''
config = ctx.meta.get('config')
return caches(config, name, list, delete, checksum)
return caches_impl(config, name, list, delete, checksum)
@experiments.command()
......@@ -618,40 +624,40 @@ def edit(ctx, name):
@experiments.command()
@click.argument('name', nargs=1)
@click.argument('names', nargs=-1)
@click.pass_context
def check(ctx, name):
def check(ctx, names):
'''Checks a local experiment for validity.
$ beat experiments check xxx
'''
config = ctx.meta.get('config')
return common.check(config.path, 'experiment', name)
return common.check(config.path, 'experiment', names)
@experiments.command()
@click.argument('name', nargs=1)
@click.argument('names', nargs=-1)
@click.option('--force', help='Performs operation regardless of conflicts',
is_flag=True)
@click.pass_context
def pull(ctx, name, force):
def pull(ctx, names, force):
'''Downloads the specified experiments from the server.
$ beat experiments pull xxx.
'''
config = ctx.meta.get('config')
with common.make_webapi(config) as webapi:
return pull_impl(webapi, config.path, name, force, 0, {})
return pull_impl(webapi, config.path, names, force, 0, {})
@experiments.command()
@click.argument('name', nargs=1)
@click.argument('names', nargs=-1)
@click.option('--force', help='Performs operation regardless of conflicts',
is_flag=True)
@click.option('--dry-run', help="Doesn't really perform the task, just "
"comments what would do", is_flag=True)
@click.pass_context
def push(ctx, name, force, dry_run):
def push(ctx, names, force, dry_run):
'''Uploads experiments to the server.
Example:
......@@ -660,7 +666,7 @@ def push(ctx, name, force, dry_run):
config = ctx.meta.get('config')
with common.make_webapi(config) as webapi:
return common.push(
webapi, config.path, 'experiment', name,
webapi, config.path, 'experiment', names,
['name', 'declaration', 'toolchain', 'description'],
{}, force, dry_run, 0
)
......@@ -710,11 +716,11 @@ def fork(ctx, src, dst):
@experiments.command()
@click.argument('name', nargs=1)
@click.argument('names', nargs=-1)
@click.option('--remote', help='Only acts on the remote copy of the experiment',
is_flag=True)
@click.pass_context
def rm(ctx, name, remote):
def rm(ctx, names, remote):
'''Deletes a local experiment (unless --remote is specified).
$ beat experiments rm xxx
......@@ -722,24 +728,24 @@ def rm(ctx, name, remote):
config = ctx.meta.get('config')
if remote:
with common.make_webapi(config) as webapi:
return common.delete_remote(webapi, 'experiment', name)
return common.delete_remote(webapi, 'experiment', names)
else:
return common.delete_local(config.path, 'experiment', name)
return common.delete_local(config.path, 'experiment', names)
@experiments.command()
@click.argument('name', nargs=1)
@click.argument('names', nargs=-1)
@click.option('--path', help='Use path to write files to disk (instead of the '
'current directory)', type=click.Path())
@click.pass_context
def draw(ctx, name, path):
def draw(ctx, names, path):
'''Creates a visual representation of the experiment.'''
config = ctx.meta.get('config')
return common.dot_diagram(config.path, 'experiment', names, path, [])
@experiments.command()
@click.argument('name', nargs=1)
@click.argument('names', nargs=-1)
@click.option('--force', help='Performs operation regardless of conflicts',
is_flag=True)
@click.option('--remote', help='Only acts on the remote copy of the experiment',
......@@ -748,11 +754,11 @@ def draw(ctx, name, path):
is_flag=True)
@click.option('--output-folder', help='<folder>', type=click.Path(exists=True))
@click.pass_context
def plot(ctx, name, force, remote, show, output_folder):
def plot(ctx, names, force, remote, show, output_folder):
'''Plots output images of the experiment.'''
config = ctx.meta.get('config')
with common.make_webapi(config) as webapi:
return plot_impl(
webapi, config, 'experiment', name, remote, show, force, 0, {},
webapi, config, 'experiment', names, remote, show, force, 0, {},
output_folder
)
......@@ -130,29 +130,29 @@ def edit(ctx, name):
@plotterparameters.command()
@click.argument('name', nargs=1)
@click.argument('names', nargs=-1)
@click.pass_context
def check(ctx, name):
def check(ctx, names):
'''Checks a local plotter for validity.
$ beat plotterparameters check xxx
'''
return common.check(ctx.meta['config'].path, 'plotterparameter', name)
return common.check(ctx.meta['config'].path, 'plotterparameter', names)
@plotterparameters.command()
@click.argument('name', nargs=1)
@click.argument('names', nargs=-1)
@click.option('--force', help='Force',
is_flag=True)
@click.pass_context
def pull(ctx, name, force):
def pull(ctx, names, force):
'''Downloads the specified plotterparameters from the server.
$ beat plotterparameters pull xxx.
'''
with common.make_webapi(ctx.meta['config']) as webapi:
name = common.make_up_remote_list(webapi, 'plotterparameter', name)
name = common.make_up_remote_list(webapi, 'plotterparameter', names)
if name is None:
return 1 #error
return pull_impl(webapi, ctx.meta['config'].path, name, force, 0, {})
......@@ -160,14 +160,14 @@ def pull(ctx, name, force):
@plotterparameters.command()
@click.argument('name', nargs=1)
@click.argument('names', nargs=-1)
@click.pass_context
def create(ctx, name):
def create(ctx, names):
'''Creates a new local plotter.
$ beat plotterparameters create xxx
'''
return common.create(ctx.meta['config'].path, 'plotterparameter', [name])
return common.create(ctx.meta['config'].path, 'plotterparameter', names)
......@@ -196,11 +196,12 @@ def fork(ctx, src, dst):
@plotterparameters.command()
@click.argument('name', nargs=1)
@click.argument('names', nargs=-1)
@click.pass_context
def rm(ctx, name):
def rm(ctx, names):
'''Deletes a local plotter.
$ beat plotterparameters rm xxx
'''
return common.delete_local(ctx.meta['config'].path, 'plotterparameter', name)
return common.delete_local(ctx.meta['config'].path, 'plotterparameter',
names)
......@@ -104,8 +104,9 @@ def pull_impl(webapi, prefix, names, force, indentation, format_cache):
for df in defaultplotters:
if name == df['plotter']:
plp_status = plotterparameters_pull(webapi, prefix, [df['parameter']], ['data', 'short_description', 'plotter'], force, indentation + 2)
return status + lb_status + df_status
code = status + lb_status + df_status
sys.exit(code)
return code
def plot_impl(webapi, prefix, names, show, force, need_data_sample, inputdata, outputimage, plotterparameter, indentation, format_cache):
"""plot sample plot from the server.
......@@ -235,8 +236,10 @@ def plot_impl(webapi, prefix, names, show, force, need_data_sample, inputdata, o
logger.info("%ssaved image `%s'...", indent, output_name)
else:
logger.error("Error: data has not been set")
sys.exit(1)
return 1
sys.exit(0)
return 0
......@@ -293,29 +296,29 @@ def edit(ctx, name):
@plotters.command()
@click.argument('name', nargs=1)
@click.argument('names', nargs=-1)
@click.pass_context
def check(ctx, name):
def check(ctx, names):
'''Checks a local plotter for validity.
$ beat plotters check xxx
'''
return common.check(ctx.meta['config'].path, 'plotter', name)
return common.check(ctx.meta['config'].path, 'plotter', names)
@plotters.command()
@click.argument('name', nargs=1)
@click.argument('names', nargs=-1)
@click.option('--force', help='Performs operation regardless of conflicts',
is_flag=True)
@click.pass_context
def pull(ctx, name, force):
def pull(ctx, names, force):
'''Downloads the specified plotters from the server.
$ beat plotters pull xxx.
'''
with common.make_webapi(ctx.meta['config']) as webapi:
name = common.make_up_remote_list(webapi, 'plotter', name)