Commit 79b4a5aa authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME
Browse files

[cache/databases/dataformats/libraries/status] Click implementation of

the commands

Reimplement cache/databases/dataformats/libraries/status existing
commands using  plugin-based command line mechanism (Click).
parent 034ca480
......@@ -25,46 +25,8 @@
# #
###############################################################################
"""Configuration manipulation and display
Usage:
%(prog)s cache clear
%(prog)s cache info [--start=<index>] [--end=<index>] [--sizes] [<path>]...
%(prog)s cache view [--start=<index>] [--end=<index>] [<path>]...
%(prog)s cache --help
Commands:
clear Deletes all available cache
info Displays information about a particular cache file
view Displays the contents of a particular cache file
Options:
-s, --sizes If set, also print the size in bytes for objects in a file. This
triggers the full file readout
--start=<i> If set, allows the user to print only a few bits of the file
--end=<i> If set, allows the user to print only a few bits of the file
-h, --help Display this screen
Examples:
To clear all available cache:
$ %(prog)s cache clear
To collect information about a particular cache file:
$ %(prog)s cache info 7f/d8/8d/a11178ac27075feaba8131fe878d6e3...
To view a particular cache file:
$ %(prog)s cache view 7f/d8/8d/a11178ac27075feaba8131fe878d6e3...
"""
import os
import click
import fnmatch
import logging
logger = logging.getLogger(__name__)
......@@ -77,119 +39,150 @@ from beat.core.utils import NumpyJSONEncoder
def get_paths(config):
func = lambda z: z.split('.', 1)[0]
retval = []
for dirname, dirs, files in os.walk(config.cache):
files = fnmatch.filter(files, '*.data') #avoid index-only files
if not files: continue
d = dirname.replace(config.cache, '').strip(os.sep)
retval += list(set([os.path.join(d, func(k)) for k in files]))
func = lambda z: z.split('.', 1)[0]
return retval
retval = []
for dirname, _, files in os.walk(config.cache):
files = fnmatch.filter(files, '*.data') #avoid index-only files
if not files:
continue
d = dirname.replace(config.cache, '').strip(os.sep)
retval += list(set([os.path.join(d, func(k)) for k in files]))
def info(config, paths, sizes, index_start, index_end):
return retval
if not paths:
paths = get_paths(config)
for path in paths:
@click.group()
@click.pass_context
@click.option('--start', type=click.INT, help='If set, allows the user to '
'print only a few bits of the file')
@click.option('--end', type=click.INT, help='If set, allows the user to '
'print only a few bits of the file')
def cache(ctx, start, end):
"""Configuration manipulation and display"""
pass
logger.info('path: %s', path)
fullpath = os.path.join(config.cache, path + '.data')
f = CachedDataSource()
status = f.setup(fullpath, config.path, index_start, index_end)
if not status:
logger.error("cannot setup data source with `%s' and prefix `%s'",
fullpath, config.path)
return 1
@cache.command()
@click.pass_context
def clear(ctx):
'''Deletes all available cache
logger.info(' dataformat: %s', f.dataformat.name)
if sizes:
counter = 0
logger.info(' index:')
for data, start, end in f:
size = len(data.pack())
counter += size
if start == end:
logger.info(' [%d] - %d bytes', start, size)
else:
logger.info(' [%d:%d] - %d bytes', start, end, size)
logger.info(' total (stripped-down) size: %d bytes', counter)
To clear all available cache:
$ %(prog)s cache clear
'''
import shutil
if os.path.isdir(ctx.meta['config'].cache):
for k in os.listdir(ctx.meta['config'].cache):
p = os.path.join(ctx.meta['config'].cache, k)
if os.path.isfile(p):
shutil.rmtree(p)
else:
raise click.ClickException("Failed to clear cache %s" % p)
else:
index = load_data_index(config.cache, path + '.data')
logger.info(' objects : %d', len(index)-1)
return 0
def view(config, paths, index_start, index_end):
if not paths:
paths = get_paths(config)
for path in paths:
raise click.ClickException("Cache directory does not exist: %s" %
ctx.meta['config'].cache)
logger.info('path: %s', path)
fullpath = os.path.join(config.cache, path + '.data')
f = CachedDataSource()
status = f.setup(fullpath, config.path, index_start, index_end)
if not status:
logger.error("cannot setup data source with `%s' and prefix `%s'",
fullpath, config.path)
return 1
@cache.command()
@click.argument('paths', nargs=-1, type=click.Path(exists=True))
@click.pass_context
@click.option('--sizes', help='If set, also print the size in bytes for '
'objects in a file. This triggers the full file readout',
is_flag=True)
def info(ctx, paths, sizes):
'''Displays information about a particular cache file
logger.info(' dataformat: %s', f.dataformat.name)
To collect information about a particular cache file:
for data, start, end in f:
logger.extra(80 * '-')
if start == end:
header = '[%d]: ' % start
else:
header = '[%d:%d]: ' % (start, end)
json_data = data.as_dict()
for name, value in json_data.items():
json_data[name] = common.stringify(value)
json_data = simplejson.dumps(json_data, indent=2,
cls=NumpyJSONEncoder).\
replace('"BEAT_LIST_DELIMITER[', '[')\
.replace(']BEAT_LIST_DELIMITER"', ']')\
.replace('"...",', '...')\
.replace('"BEAT_LIST_SIZE(', '(')\
.replace(')BEAT_LIST_SIZE"', ')')
logger.info(header + json_data)
return 0
def process(args):
$ %(prog)s cache info 7f/d8/8d/a11178ac27075feaba8131fe878d6e3...
'''
config = ctx.meta['config']
index_start = int(ctx.meta['start']) if 'start' in ctx.meta else None
index_end = int(ctx.meta['end']) if 'end' in ctx.meta else None
if not paths:
paths = get_paths(config)
for path in paths:
logger.info('path: %s', path)
fullpath = os.path.join(config.cache, path + '.data')
f = CachedDataSource()
status = f.setup(fullpath, config.path, index_start, index_end)
if not status:
logger.error("cannot setup data source with `%s' and prefix `%s'",
fullpath, config.path)
return 1
logger.info(' dataformat: %s', f.dataformat.name)
if sizes:
counter = 0
logger.info(' index:')
for data, start, end in f:
size = len(data.pack())
counter += size
if start == end:
logger.info(' [%d] - %d bytes', start, size)
else:
logger.info(' [%d:%d] - %d bytes', start, end, size)
logger.info(' total (stripped-down) size: %d bytes', counter)
start = int(args['--start']) if args['--start'] is not None else None
end = int(args['--end']) if args['--end'] is not None else None
else:
index = load_data_index(config.cache, path + '.data')
logger.info(' objects : %d', len(index)-1)
if args['clear']:
import shutil
for k in os.listdir(args['config'].cache):
p = os.path.join(args['config'].cache, k)
shutil.rmtree(p)
return 0
elif args['info']:
return info(args['config'], args['<path>'], args['--sizes'], start, end)
@cache.command()
@click.argument('paths', nargs=-1)
@click.pass_context
def view(ctx, paths):
'''Displays information about a particular cache file
elif args['view']:
return view(args['config'], args['<path>'], start, end)
To view a particular cache file:
# Should not happen
logger.error("unrecognized `cache' subcommand")
return 1
$ %(prog)s cache view 7f/d8/8d/a11178ac27075feaba8131fe878d6e3...
'''
config = ctx.meta['config']
index_start = int(ctx.meta['start']) if 'start' in ctx.meta else None
index_end = int(ctx.meta['end']) if 'end' in ctx.meta else None
if not paths:
paths = get_paths(config)
for path in paths:
logger.info('path: %s', path)
fullpath = os.path.join(config.cache, path + '.data')
f = CachedDataSource()
status = f.setup(fullpath, config.path, index_start, index_end)
if not status:
logger.error("cannot setup data source with `%s' and prefix `%s'",
fullpath, config.path)
return 1
logger.info(' dataformat: %s', f.dataformat.name)
for data, start, end in f:
logger.extra(80 * '-')
if start == end:
header = '[%d]: ' % start
else:
header = '[%d:%d]: ' % (start, end)
json_data = data.as_dict()
for name, value in json_data.items():
json_data[name] = common.stringify(value)
json_data = simplejson.dumps(
json_data, indent=2,
cls=NumpyJSONEncoder).\
replace('"BEAT_LIST_DELIMITER[', '[')\
.replace(']BEAT_LIST_DELIMITER"', ']')\
.replace('"...",', '...')\
.replace('"BEAT_LIST_SIZE(', '(')\
.replace(')BEAT_LIST_SIZE"', ')')
logger.info(header + json_data)
......@@ -25,75 +25,8 @@
# #
###############################################################################
"""Usage:
%(prog)s databases list [--remote]
%(prog)s databases path [<name>]...
%(prog)s databases edit <name>...
%(prog)s databases check [<name>]...
%(prog)s databases pull [--force] [<name>]...
%(prog)s databases push [--force] [--dry-run] [<name>]...
%(prog)s databases diff <name>
%(prog)s databases status
%(prog)s databases version <name>
%(prog)s databases index [--list | --delete | --checksum] [--uid=<uid>] [--db-root=<path>] [--docker] [<name>]...
%(prog)s databases view [--exclude=<output>] [--uid=<uid>] [--db-root=<path>] [--docker] <set_name>
%(prog)s databases --help
Arguments:
<name> Database name formated as "<database>/<version>"
<set_name> Set formatted as "<database>/<version>/<protocol>/<set>"
Commands:
list Lists all the databases available on the platform
path Displays local path of databases files
edit Edit local database file
check Checks a local database for validity
pull Downloads the specified databases from the server
push Uploads databases to the server (must provide a valid admin token)
diff Shows changes between the local database and the remote version
status Shows (editing) status for all available databases
version Creates a new version of an existing database
index Indexes all outputs (of all sets) of a database.
view View the data of the specified dataset.
Options:
--help Display this screen
--remote Only acts on the remote copy of the database
--exclude=<output> When viewing, excludes this output
--list List index files matching output if they exist
--delete Delete index files matching output if they exist (also,
recursively deletes empty directories)
--checksum Checksums index files
Examples:
To list all existing databases on your local prefix:
$ %(prog)s db list
To view the contents of a specific set
$ %(prog)s db view simple/1/protocol/set
To index the contents of a database
$ %(prog)s db index simple/1
To index the contents of a protocol on a database
$ %(prog)s db index simple/1/double
To index the contents of a set in a protocol on a database
$ %(prog)s db index simple/1/double/double
"""
import os
import click
import glob
import random
import zmq
......@@ -333,7 +266,7 @@ def start_db_container(configuration, cmd, host,
# ----------------------------------------------------------
def pull(webapi, prefix, names, force, indentation, format_cache):
def pull_impl(webapi, prefix, names, force, indentation, format_cache):
"""Copies databases (and required dataformats) from the server.
Parameters:
......@@ -631,78 +564,194 @@ def view_outputs(configuration, dataset_name, excluded_outputs=None, uid=None,
# ----------------------------------------------------------
def process(args):
uid = int(args['--uid']) if args['--uid'] is not None else None
configuration = args['config']
db_names = args['<name>']
@click.group()
@click.pass_context
def db(ctx):
"""Database commands"""
pass
if args['list']:
if args['--remote']:
with common.make_webapi(configuration) as webapi:
return common.display_remote_list(webapi, 'database')
else:
return common.display_local_list(configuration.path, 'database')
elif args['path']:
return common.display_local_path(configuration.path, 'database', db_names)
@db.command()
@click.option('--remote', help='Only acts on the remote copy of the database.',
is_flag=True)
@click.pass_context
def list(ctx, remote):
'''Lists all the databases available on the platform.
elif args['edit']:
return common.edit_local_file(configuration.path, configuration.editor, 'database', db_names[0])
To list all existing databases on your local prefix:
elif args['check']:
return common.check(configuration.path, 'database', db_names)
elif args['pull']:
with common.make_webapi(configuration) as webapi:
return pull(webapi, configuration.path, db_names,
args['--force'], 0, {})
elif args['push']:
$ beat db list
'''
configuration = ctx.meta['config']
if remote:
with common.make_webapi(configuration) as webapi:
return common.push(webapi, configuration.path, 'database',
db_names, ['name', 'declaration',
'code', 'description'],
{}, args['--force'], args['--dry-run'], 0)
return common.display_remote_list(webapi, 'database')
else:
return common.display_local_list(configuration.path, 'database')
@db.command()
@click.argument('db_names', nargs=-1)
@click.pass_context
def check(ctx, db_names):
'''Checks a local database for validity.
$ beat db check [<name>]...
<name>:
Database name formatted as "<database>/<version>"
'''
return common.check(ctx.meta['config'].path, 'database', db_names)
@db.command()
@click.argument('db_names', nargs=-1)
@click.option('--force', help='Force',
is_flag=True)
@click.pass_context
def pull(ctx, db_names, force):
'''Downloads the specified databases from the server.
$ beat db pull [<name>]...
<name>:
Database name formatted as "<database>/<version>"
'''
configuration = ctx.meta['config']
with common.make_webapi(configuration) as webapi:
return pull_impl(webapi, configuration.path, db_names, force, 0, {})
@db.command()
@click.argument('db_names', nargs=-1)
@click.option('--force', help='Force',
is_flag=True)
@click.option('--dry-run', help='Dry run',
is_flag=True)
@click.pass_context
def push(ctx, db_names, force, dry_run):
'''Uploads databases to the server (must provide a valid admin token).
$ beat db push [<name>]...
<name>:
Database name formatted as "<database>/<version>"
'''
configuration = ctx.meta['config']
with common.make_webapi(configuration) as webapi:
return common.push(webapi, configuration.path, 'database',
db_names, ['name', 'declaration',
'code', 'description'],
{}, force, dry_run, 0)
@db.command()
@click.argument('db_names', nargs=-1)
@click.pass_context
def diff(ctx, db_names):
'''Shows changes between the local database and the remote version.
$ beat db diff [<name>]...
<name>:
Database name formatted as "<database>/<version>"
'''
configuration = ctx.meta['config']
if len(db_names) < 1:
raise click.ClickException("Requires at least one database name")
with common.make_webapi(configuration) as webapi:
return common.diff(webapi, configuration.path, 'database',
db_names[0],
['declaration', 'code', 'description'])
@db.command()
@click.pass_context
def status(ctx):
'''Shows (editing) status for all available databases'''
configuration = ctx.meta['config']
with common.make_webapi(configuration) as webapi:
return common.status(webapi, configuration.path, 'database')[0]
@db.command()
@click.argument('db_names', nargs=-1)
@click.pass_context
def version(ctx, db_names):
'''Creates a new version of an existing database.
$ beat db push [<name>]...
<name>:
Database name formatted as "<database>/<version>"
'''
configuration = ctx.meta['config']
if len(db_names) < 1:
raise click.ClickException("Requires at least one database name")
return common.new_version(configuration.path, 'database', db_names[0])
@db.command()
@click.argument('db_names', nargs=-1)
@click.option('--list', help='List index files matching output if they exist',
is_flag=True)
@click.option('--delete', help='Delete index files matching output if they '
'exist (also, recursively deletes empty directories)',
is_flag=True)
@click.option('--checksum', help='Checksums index files', is_flag=True)
@click.option('--uid', type=click.INT, default=None)
@click.option('--db-root', help="Database root")
@click.option('--docker', is_flag=True)
@click.pass_context
def index(ctx, db_names, list, delete, checksum, uid, db_root, docker):
'''Indexes all outputs (of all sets) of a database.
To index the contents of a database
$ beat db index simple/1
To index the contents of a protocol on a database
$ beat db index simple/1/double
To index the contents of a set in a protocol on a database
$ beat db index simple/1/double/double
'''
configuration = ctx.meta['config']
if list:
return list_index_files(configuration, db_names)
elif delete:
return delete_index_files(configuration, db_names)
elif checksum:
return index_outputs(
configuration, db_names, uid=uid,
db_root=db_root, docker=docker
)
elif args['diff']:
with common.make_webapi(configuration) as webapi:
return common.diff(webapi, configuration.path, 'database',
db_names[0],
['declaration', 'code', 'description'])
elif args['status']:
with common.make_webapi(configuration) as webapi:
return common.status(webapi, configuration.path, 'database')[0]
elif args['version']:
return common.new_version(configuration.path, 'database',
db_names[0])
elif args['view']:
if args['--exclude']:
return view_outputs(configuration, args['<set_name>'],
args['--exclude'],
uid=uid,
db_root=args['--db-root'],
docker=args['--docker'])
else:
return view_outputs(configuration, args['<set_name>'],
uid=uid,
db_root=args['--db-root'],
docker=args['--docker'])
elif args['index']:
if args['--list']:
return list_index_files(configuration, db_names)
elif args['--delete']:
return delete_index_files(configuration, db_names)