Commit 79b4a5aa authored by Theophile GENTILHOMME's avatar Theophile GENTILHOMME

[cache/databases/dataformats/libraries/status] Click implementation of

the commands

Reimplement cache/databases/dataformats/libraries/status existing
commands using  plugin-based command line mechanism (Click).
parent 034ca480
......@@ -25,46 +25,8 @@
# #
###############################################################################
"""Configuration manipulation and display
Usage:
%(prog)s cache clear
%(prog)s cache info [--start=<index>] [--end=<index>] [--sizes] [<path>]...
%(prog)s cache view [--start=<index>] [--end=<index>] [<path>]...
%(prog)s cache --help
Commands:
clear Deletes all available cache
info Displays information about a particular cache file
view Displays the contents of a particular cache file
Options:
-s, --sizes If set, also print the size in bytes for objects in a file. This
triggers the full file readout
--start=<i> If set, allows the user to print only a few bits of the file
--end=<i> If set, allows the user to print only a few bits of the file
-h, --help Display this screen
Examples:
To clear all available cache:
$ %(prog)s cache clear
To collect information about a particular cache file:
$ %(prog)s cache info 7f/d8/8d/a11178ac27075feaba8131fe878d6e3...
To view a particular cache file:
$ %(prog)s cache view 7f/d8/8d/a11178ac27075feaba8131fe878d6e3...
"""
import os
import click
import fnmatch
import logging
logger = logging.getLogger(__name__)
......@@ -77,119 +39,150 @@ from beat.core.utils import NumpyJSONEncoder
def get_paths(config):
func = lambda z: z.split('.', 1)[0]
retval = []
for dirname, dirs, files in os.walk(config.cache):
files = fnmatch.filter(files, '*.data') #avoid index-only files
if not files: continue
d = dirname.replace(config.cache, '').strip(os.sep)
retval += list(set([os.path.join(d, func(k)) for k in files]))
func = lambda z: z.split('.', 1)[0]
return retval
retval = []
for dirname, _, files in os.walk(config.cache):
files = fnmatch.filter(files, '*.data') #avoid index-only files
if not files:
continue
d = dirname.replace(config.cache, '').strip(os.sep)
retval += list(set([os.path.join(d, func(k)) for k in files]))
def info(config, paths, sizes, index_start, index_end):
return retval
if not paths:
paths = get_paths(config)
for path in paths:
@click.group()
@click.pass_context
@click.option('--start', type=click.INT, help='If set, allows the user to '
'print only a few bits of the file')
@click.option('--end', type=click.INT, help='If set, allows the user to '
'print only a few bits of the file')
def cache(ctx, start, end):
"""Configuration manipulation and display"""
pass
logger.info('path: %s', path)
fullpath = os.path.join(config.cache, path + '.data')
f = CachedDataSource()
status = f.setup(fullpath, config.path, index_start, index_end)
if not status:
logger.error("cannot setup data source with `%s' and prefix `%s'",
fullpath, config.path)
return 1
@cache.command()
@click.pass_context
def clear(ctx):
'''Deletes all available cache
logger.info(' dataformat: %s', f.dataformat.name)
if sizes:
counter = 0
logger.info(' index:')
for data, start, end in f:
size = len(data.pack())
counter += size
if start == end:
logger.info(' [%d] - %d bytes', start, size)
else:
logger.info(' [%d:%d] - %d bytes', start, end, size)
logger.info(' total (stripped-down) size: %d bytes', counter)
To clear all available cache:
$ %(prog)s cache clear
'''
import shutil
if os.path.isdir(ctx.meta['config'].cache):
for k in os.listdir(ctx.meta['config'].cache):
p = os.path.join(ctx.meta['config'].cache, k)
if os.path.isfile(p):
shutil.rmtree(p)
else:
raise click.ClickException("Failed to clear cache %s" % p)
else:
index = load_data_index(config.cache, path + '.data')
logger.info(' objects : %d', len(index)-1)
return 0
def view(config, paths, index_start, index_end):
if not paths:
paths = get_paths(config)
for path in paths:
raise click.ClickException("Cache directory does not exist: %s" %
ctx.meta['config'].cache)
logger.info('path: %s', path)
fullpath = os.path.join(config.cache, path + '.data')
f = CachedDataSource()
status = f.setup(fullpath, config.path, index_start, index_end)
if not status:
logger.error("cannot setup data source with `%s' and prefix `%s'",
fullpath, config.path)
return 1
@cache.command()
@click.argument('paths', nargs=-1, type=click.Path(exists=True))
@click.pass_context
@click.option('--sizes', help='If set, also print the size in bytes for '
'objects in a file. This triggers the full file readout',
is_flag=True)
def info(ctx, paths, sizes):
'''Displays information about a particular cache file
logger.info(' dataformat: %s', f.dataformat.name)
To collect information about a particular cache file:
for data, start, end in f:
logger.extra(80 * '-')
if start == end:
header = '[%d]: ' % start
else:
header = '[%d:%d]: ' % (start, end)
json_data = data.as_dict()
for name, value in json_data.items():
json_data[name] = common.stringify(value)
json_data = simplejson.dumps(json_data, indent=2,
cls=NumpyJSONEncoder).\
replace('"BEAT_LIST_DELIMITER[', '[')\
.replace(']BEAT_LIST_DELIMITER"', ']')\
.replace('"...",', '...')\
.replace('"BEAT_LIST_SIZE(', '(')\
.replace(')BEAT_LIST_SIZE"', ')')
logger.info(header + json_data)
return 0
def process(args):
$ %(prog)s cache info 7f/d8/8d/a11178ac27075feaba8131fe878d6e3...
'''
config = ctx.meta['config']
index_start = int(ctx.meta['start']) if 'start' in ctx.meta else None
index_end = int(ctx.meta['end']) if 'end' in ctx.meta else None
if not paths:
paths = get_paths(config)
for path in paths:
logger.info('path: %s', path)
fullpath = os.path.join(config.cache, path + '.data')
f = CachedDataSource()
status = f.setup(fullpath, config.path, index_start, index_end)
if not status:
logger.error("cannot setup data source with `%s' and prefix `%s'",
fullpath, config.path)
return 1
logger.info(' dataformat: %s', f.dataformat.name)
if sizes:
counter = 0
logger.info(' index:')
for data, start, end in f:
size = len(data.pack())
counter += size
if start == end:
logger.info(' [%d] - %d bytes', start, size)
else:
logger.info(' [%d:%d] - %d bytes', start, end, size)
logger.info(' total (stripped-down) size: %d bytes', counter)
start = int(args['--start']) if args['--start'] is not None else None
end = int(args['--end']) if args['--end'] is not None else None
else:
index = load_data_index(config.cache, path + '.data')
logger.info(' objects : %d', len(index)-1)
if args['clear']:
import shutil
for k in os.listdir(args['config'].cache):
p = os.path.join(args['config'].cache, k)
shutil.rmtree(p)
return 0
elif args['info']:
return info(args['config'], args['<path>'], args['--sizes'], start, end)
@cache.command()
@click.argument('paths', nargs=-1)
@click.pass_context
def view(ctx, paths):
'''Displays information about a particular cache file
elif args['view']:
return view(args['config'], args['<path>'], start, end)
To view a particular cache file:
# Should not happen
logger.error("unrecognized `cache' subcommand")
return 1
$ %(prog)s cache view 7f/d8/8d/a11178ac27075feaba8131fe878d6e3...
'''
config = ctx.meta['config']
index_start = int(ctx.meta['start']) if 'start' in ctx.meta else None
index_end = int(ctx.meta['end']) if 'end' in ctx.meta else None
if not paths:
paths = get_paths(config)
for path in paths:
logger.info('path: %s', path)
fullpath = os.path.join(config.cache, path + '.data')
f = CachedDataSource()
status = f.setup(fullpath, config.path, index_start, index_end)
if not status:
logger.error("cannot setup data source with `%s' and prefix `%s'",
fullpath, config.path)
return 1
logger.info(' dataformat: %s', f.dataformat.name)
for data, start, end in f:
logger.extra(80 * '-')
if start == end:
header = '[%d]: ' % start
else:
header = '[%d:%d]: ' % (start, end)
json_data = data.as_dict()
for name, value in json_data.items():
json_data[name] = common.stringify(value)
json_data = simplejson.dumps(
json_data, indent=2,
cls=NumpyJSONEncoder).\
replace('"BEAT_LIST_DELIMITER[', '[')\
.replace(']BEAT_LIST_DELIMITER"', ']')\
.replace('"...",', '...')\
.replace('"BEAT_LIST_SIZE(', '(')\
.replace(')BEAT_LIST_SIZE"', ')')
logger.info(header + json_data)
This diff is collapsed.
......@@ -25,48 +25,8 @@
# #
###############################################################################
"""Usage:
%(prog)s dataformats list [--remote]
%(prog)s dataformats path [<name>]...
%(prog)s dataformats edit <name>...
%(prog)s dataformats check [<name>]...
%(prog)s dataformats pull [--force] [<name>]...
%(prog)s dataformats push [--force] [--dry-run] [<name>]...
%(prog)s dataformats diff <name>
%(prog)s dataformats status
%(prog)s dataformats create <name>...
%(prog)s dataformats version <name>
%(prog)s dataformats fork <src> <dst>
%(prog)s dataformats rm [--remote] <name>...
%(prog)s dataformats --help
Commands:
list Lists all the dataformats available on the platform
path Displays local path of dataformats files
edit Edit local dataformat file
check Checks a local dataformat for validity
pull Downloads the specified dataformats from the server
push Uploads dataformats to the server
diff Shows changes between the local dataformat and the remote version
status Shows (editing) status for all available dataformats
create Creates a new local dataformat
version Creates a new version of an existing dataformat
fork Forks a local dataformat
rm Deletes a local dataformat (unless --remote is specified)
Options:
--force Performs operation regardless of conflicts
--dry-run Doesn't really perform the task, just comments what would do
--remote Only acts on the remote copy of the dataformat
--help Display this screen
"""
import logging
logger = logging.getLogger(__name__)
import click
import oset
......@@ -74,8 +34,9 @@ from . import common
from beat.core import dataformat
LOGGER = logging.getLogger(__name__)
def pull(webapi, prefix, names, force, indentation, cache):
def pull_impl(webapi, prefix, names, force, indentation, cache):
"""Copies dataformats (recursively) from the server.
Data formats are particularly tricky to download because of their recursive
......@@ -139,73 +100,189 @@ def pull(webapi, prefix, names, force, indentation, cache):
dataformats |= obj.referenced.keys()
except Exception as e:
logger.error("loading `%s': %s...", name, str(e))
LOGGER.error("loading `%s': %s...", name, str(e))
cache[name] = None
# recurse until done
return pull(webapi, prefix, dataformats, force, 2+indentation, cache)
def process(args):
if args['list']:
if args['--remote']:
with common.make_webapi(args['config']) as webapi:
return common.display_remote_list(webapi, 'dataformat')
else:
return common.display_local_list(args['config'].path, 'dataformat')
elif args['path']:
return common.display_local_path(args['config'].path, 'dataformat', args['<name>'])
elif args['edit']:
return common.edit_local_file(args['config'].path, args['config'].editor, 'dataformat', args['<name>'][0])
elif args['check']:
return common.check(args['config'].path, 'dataformat', args['<name>'])
elif args['pull']:
with common.make_webapi(args['config']) as webapi:
args['<name>'] = common.make_up_remote_list(webapi, 'dataformat',
args['<name>'])
if args['<name>'] is None: return 1 #error
return pull(webapi, args['config'].path, args['<name>'],
args['--force'], 0, {})
elif args['push']:
with common.make_webapi(args['config']) as webapi:
return common.push(webapi, args['config'].path, 'dataformat',
args['<name>'], ['name', 'declaration', 'description'], {},
args['--force'], args['--dry-run'], 0)
elif args['diff']:
with common.make_webapi(args['config']) as webapi:
return common.diff(webapi, args['config'].path, 'dataformat',
args['<name>'][0], ['declaration', 'description'])
elif args['status']:
with common.make_webapi(args['config']) as webapi:
return common.status(webapi, args['config'].path, 'dataformat')[0]
elif args['create']:
return common.create(args['config'].path, 'dataformat', args['<name>'])
elif args['version']:
return common.new_version(args['config'].path, 'dataformat',
args['<name>'][0])
elif args['fork']:
return common.fork(args['config'].path, 'dataformat',
args['<src>'], args['<dst>'])
elif args['rm']:
if args['--remote']:
with common.make_webapi(args['config']) as webapi:
return common.delete_remote(webapi, 'dataformat', args['<name>'])
else:
return common.delete_local(args['config'].path,
'dataformat', args['<name>'])
# Should not happen
logger.error("unrecognized `dataformats' subcommand")
return 1
return pull_impl(webapi, prefix, dataformats, force, 2+indentation, cache)
@click.group()
@click.pass_context
def df(ctx):
"""Configuration manipulation of data formats"""
pass
@df.command()
@click.option('--remote', help='Only acts on the remote copy of the dataformat',
is_flag=True)
@click.pass_context
def list(ctx, remote):
'''Lists all the dataformats available on the platform
Example:
$ beat df list --remote
'''
if remote:
with common.make_webapi(ctx.meta['config']) as webapi:
return common.display_remote_list(webapi, 'dataformat')
else:
return common.display_local_list(ctx.meta['config'].path, 'dataformat')
@df.command()
@click.argument('names', nargs=-1)
@click.pass_context
def path(ctx, names):
'''Displays local path of dataformats files
Example:
$ beat dataformats path xxx
'''
return common.display_local_path(ctx.meta['config'].path, 'dataformat', names)
@df.command()
@click.argument('name', nargs=1)
@click.pass_context
def edit(ctx, name):
'''Edit local dataformat file
Example:
$ beat df edit xxx
'''
return common.edit_local_file(ctx.meta['config'].path,
ctx.meta['config'].editor, 'dataformat',
name)
@df.command()
@click.argument('names', nargs=-1)
@click.pass_context
def check(ctx, names):
'''Checks a local dataformat for validity
Example:
$ beat df check xxx
'''
return common.check(ctx.meta['config'].path, 'dataformat', names)
@df.command()
@click.argument('name', nargs=1)
@click.option('--force', help='Performs operation regardless of conflicts',
is_flag=True)
@click.pass_context
def pull(ctx, name, force):
'''Downloads the specified dataformats from the server
Example:
$ beat df pull --force yyy
'''
with common.make_webapi(ctx.meta['config']) as webapi:
name = common.make_up_remote_list(webapi, 'dataformat', name)
if name is None:
return 1 #error
return pull_impl(webapi, ctx.meta['config'].path, name,force, 0, {})
@df.command()
@click.argument('name', nargs=1)
@click.option('--force', help='Performs operation regardless of conflicts',
is_flag=True)
@click.option('--dry-run', help="Doesn't really perform the task, just "
"comments what would do", is_flag=True)
@click.pass_context
def push(ctx, name, force, dry_run):
'''Uploads dataformats to the server
Example:
$ beat df push --dry-run yyy
'''
with common.make_webapi(ctx.meta['config']) as webapi:
return common.push(
webapi, ctx.meta['config'].path, 'dataformat',
name, ['name', 'declaration', 'description'], {},
force, dry_run, 0
)
@df.command()
@click.argument('name', nargs=1)
@click.pass_context
def diff(ctx, name):
'''Shows changes between the local dataformat and the remote version
Example:
$ beat df diff xxx
'''
with common.make_webapi(ctx.meta.get('config')) as webapi:
return common.diff(webapi, ctx.meta.get('config').path, 'dataformat',
name, ['declaration', 'description'])
@df.command()
@click.pass_context
def status(ctx):
'''Shows (editing) status for all available dataformats
Example:
$ beat df status
'''
with common.make_webapi(ctx.meta['config']) as webapi:
return common.status(webapi, ctx.meta['config'].path, 'dataformat')[0]
@df.command()
@click.argument('names', nargs=-1)
@click.pass_context
def create(ctx, names):
'''Creates a new local dataformat
Example:
$ beat df create xxx
'''
return common.create(ctx.meta['config'].path, 'dataformat', names)
@df.command()
@click.argument('name', nargs=1)
@click.pass_context
def version(ctx, name):
'''Creates a new version of an existing dataformat
Example:
$ beat df version xxx
'''
return common.new_version(ctx.meta['config'].path, 'dataformat', name)
@df.command()
@click.argument('src', nargs=1)
@click.argument('dst', nargs=1)
@click.pass_context
def fork(ctx, src, dst):
'''Forks a local dataformat
Example:
$ beat df fork xxx yyy
'''
return common.fork(ctx.meta['config'].path, 'dataformat', src, dst)
@df.command()
@click.argument('name', nargs=1)
@click.option('--remote', help='Only acts on the remote copy of the dataformat',
is_flag=True)
@click.pass_context
def rm(ctx, name, remote):
'''Deletes a local dataformat (unless --remote is specified)
Example:
$ beat df rm xxx
'''
if remote:
with common.make_webapi(ctx.meta['config']) as webapi:
return common.delete_remote(webapi, 'dataformat', name)
else:
return common.delete_local(ctx.meta['config'].path, 'dataformat', name)
......@@ -66,16 +66,16 @@ Options:
"""
import logging
logger = logging.getLogger(__name__)
import click
import oset
from . import common
from beat.core import library
logger = logging.getLogger(__name__)
def pull(webapi, prefix, names, force, indentation, cache):
def pull_impl(webapi, prefix, names, force, indentation, cache):
"""Copies libraries (and dependent libraries) from the server.
Parameters:
......@@ -138,64 +138,180 @@ def pull(webapi, prefix, names, force, indentation, cache):
cache[name] = None
# recurse until done
return pull(webapi, prefix, libraries, force, 2+indentation, cache)
def process(args):
if args['list']:
if args['--remote']:
with common.make_webapi(args['config']) as webapi:
return common.display_remote_list(webapi, 'library')
else:
return common.display_local_list(args['config'].path, 'library')
elif args['path']:
return common.display_local_path(args['config'].path, 'library', args['<name>'])
elif args['edit']:
return common.edit_local_file(args['config'].path, args['config'].editor, 'library', args['<name>'][0])
elif args['check']:
return common.check(args['config'].path, 'library', args['<name>'])
elif args['pull']:
with common.make_webapi(args['config']) as webapi:
return pull(webapi, args['config'].path, args['<name>'],
args['--force'], 0, {})
elif args['push']:
with common.make_webapi(args['config']) as webapi:
return common.push(webapi, args['config'].path, 'library',
args['<name>'], ['name', 'declaration', 'code', 'description'],
{}, args['--force'], args['--dry-run'], 0)
elif args['diff']:
with common.make_webapi(args['config']) as webapi:
return common.diff(webapi, args['config'].path, 'library',
args['<name>'][0], ['declaration', 'code', 'description'])
elif args['status']:
with common.make_webapi(args['config']) as webapi:
return common.status(webapi, args['config'].path, 'library')[0]
elif args['create']:
return common.create(args['config'].path, 'library', args['<name>'])
elif args['version']:
return common.new_version(args['config'].path, 'library',
args['<name>'][0])
elif args['fork']:
return common.fork(args['config'].path, 'library',
args['<src>'], args['<dst>'])
elif args['rm']:
if args['--remote']:
with common.make_webapi(args['config']) as webapi:
return common.delete_remote(webapi, 'library', args['<name>'])
else:
return common.delete_local(args['config'].path, 'library',
args['<name>'])
return 1
return pull_impl(webapi, prefix, libraries, force, 2+indentation, cache)
@click.group()
@click.pass_context
def libraries(ctx):