Commit 626b4a8c authored by Samuel GAIST's avatar Samuel GAIST
Browse files

Merge branch 'cli' into '1.4.x'

use click for command lines

See merge request !25
parents e79a64a7 2e565dec
Pipeline #21210 passed with stages
in 36 minutes and 32 seconds
......@@ -25,68 +25,8 @@
# #
###############################################################################
"""Usage:
%(prog)s algorithms list [--remote]
%(prog)s algorithms check [<name>]...
%(prog)s algorithms path [<name>]...
%(prog)s algorithms edit <name>...
%(prog)s algorithms pull [--force] [<name>]...
%(prog)s algorithms push [--force] [--dry-run] [<name>]...
%(prog)s algorithms diff <name>
%(prog)s algorithms status
%(prog)s algorithms create <name>...
%(prog)s algorithms version <name>
%(prog)s algorithms fork <src> <dst>
%(prog)s algorithms rm [--remote] <name>...
%(prog)s algorithms execute <instructions>
%(prog)s algorithms execute --examples
%(prog)s algorithms --help
Commands:
list Lists all the algorithms available on the platform
path Displays local path of algorithm files
edit Edit local algorithm file
check Checks a local algorithm for validity
pull Downloads the specified algorithms from the server
push Uploads algorithms to the server
diff Shows changes between the local algorithm and the remote version
status Shows (editing) status for all available algorithms
create Creates a new local algorithm
version Creates a new version of an existing algorithm
fork Forks a local algorithm
rm Deletes a local algorithm (unless --remote is specified)
execute Execute an algorithm following instructions in a JSON file
Options:
--force Performs operation regardless of conflicts
--dry-run Doesn't really perform the task, just comments what would do
--remote Only acts on the remote copy of the algorithm
--examples Display some example JSON instruction files
--help Display this screen
Arguments for 'execute':
<instructions> JSON file containing the instructions
Examples:
To list all algorithms available locally:
$ %(prog)s al list
To list all algorithms available at the platform:
$ %(prog)s al list --remote
"""
import click
import logging
logger = logging.getLogger(__name__)
import os
import sys
import docopt
......@@ -94,17 +34,19 @@ import simplejson as json
from . import common
from beat.cmdline.scripts.click_helper import AliasedGroup
from beat.core import algorithm
from beat.core.execution import DockerExecutor
from beat.core.dock import Host
from beat.core import hash
from beat.backend.python.database import Storage as DatabaseStorage
from beat.backend.python.algorithm import Storage as AlgorithmStorage
logger = logging.getLogger(__name__)
def pull(webapi, prefix, names, force, indentation, format_cache, lib_cache):
def pull_impl(webapi, prefix, names, force, indentation, format_cache, lib_cache):
"""Copies algorithms (and required libraries/dataformats) from the server.
Parameters:
......@@ -143,8 +85,8 @@ def pull(webapi, prefix, names, force, indentation, format_cache, lib_cache):
"""
from .dataformats import pull as dataformats_pull
from .libraries import pull as libraries_pull
from .dataformats import pull_impl as dataformats_pull
from .libraries import pull_impl as libraries_pull
status, names = common.pull(webapi, prefix, 'algorithm', names,
['declaration', 'code', 'description'], force, indentation)
......@@ -255,7 +197,7 @@ To execute an analyzer:
def execute(prefix, cache, instructions_file):
def execute_impl(prefix, cache, instructions_file):
try:
# Load the JSON configuration
if not os.path.exists(instructions_file):
......@@ -346,70 +288,206 @@ def execute(prefix, cache, instructions_file):
def process(args):
@click.group(cls=AliasedGroup)
@click.pass_context
def algorithms(ctx):
"""Configuration and manipulation of algorithms"""
pass
@algorithms.command()
@click.option('--remote', help='Only acts on the remote copy of the algorithm',
is_flag=True)
@click.pass_context
def list(ctx, remote):
'''Lists all the algorithms available on the platform
if args['list']:
if args['--remote']:
with common.make_webapi(args['config']) as webapi:
return common.display_remote_list(webapi, 'algorithm')
else:
return common.display_local_list(args['config'].path, 'algorithm')
Example:
$ beat algorithms list --remote
'''
if remote:
with common.make_webapi(ctx.meta['config']) as webapi:
return common.display_remote_list(webapi, 'algorithm')
else:
return common.display_local_list(ctx.meta['config'].path, 'algorithm')
@algorithms.command()
@click.argument('names', nargs=-1)
@click.pass_context
def path(ctx, names):
'''Displays local path of algorithm files
elif args['path']:
return common.display_local_path(args['config'].path, 'algorithm', args['<name>'])
Example:
$ beat algorithms path xxx
'''
return common.display_local_path(ctx.meta['config'].path, 'algorithm', names)
elif args['edit']:
return common.edit_local_file(args['config'].path, args['config'].editor, 'algorithm', args['<name>'][0])
elif args['check']:
return common.check(args['config'].path, 'algorithm', args['<name>'])
@algorithms.command()
@click.argument('name', nargs=1)
@click.pass_context
def edit(ctx, name):
'''Edit local algorithm file
elif args['pull']:
with common.make_webapi(args['config']) as webapi:
return pull(webapi, args['config'].path, args['<name>'],
args['--force'], 0, {}, {})
Example:
$ beat algorithms edit xxx
'''
return common.edit_local_file(ctx.meta['config'].path,
ctx.meta['config'].editor, 'algorithm',
name)
elif args['push']:
with common.make_webapi(args['config']) as webapi:
return common.push(webapi, args['config'].path, 'algorithm',
args['<name>'], ['name', 'declaration', 'code', 'description'],
{}, args['--force'], args['--dry-run'], 0)
elif args['diff']:
with common.make_webapi(args['config']) as webapi:
return common.diff(webapi, args['config'].path, 'algorithm',
args['<name>'][0], ['declaration', 'code', 'description'])
@algorithms.command()
@click.argument('name', nargs=-1)
@click.pass_context
def check(ctx, name):
'''Checks a local algorithm for validity
elif args['status']:
with common.make_webapi(args['config']) as webapi:
return common.status(webapi, args['config'].path, 'algorithm')[0]
Example:
$ beat algorithms check xxx
'''
return common.check(ctx.meta['config'].path, 'algorithm', name)
elif args['create']:
return common.create(args['config'].path, 'algorithm', args['<name>'])
elif args['version']:
return common.new_version(args['config'].path, 'algorithm',
args['<name>'][0])
elif args['fork']:
return common.fork(args['config'].path, 'algorithm',
args['<src>'], args['<dst>'])
@algorithms.command()
@click.argument('name', nargs=-1)
@click.option('--force', help='Performs operation regardless of conflicts',
is_flag=True)
@click.pass_context
def pull(ctx, name, force):
'''Downloads the specified algorithms from the server
elif args['execute']:
if args['--examples']:
print_examples()
return 0
Example:
$ beat algorithms pull --force yyy
'''
with common.make_webapi(ctx.meta['config']) as webapi:
return pull_impl(webapi, ctx.meta['config'].path, name, force, 0, {}, {})
return execute(args['config'].path, args['config'].cache, args['<instructions>'])
elif args['rm']:
if args['--remote']:
with common.make_webapi(args['config']) as webapi:
return common.delete_remote(webapi, 'algorithm', args['<name>'])
else:
return common.delete_local(args['config'].path, 'algorithm',
args['<name>'])
# Should not happen
logger.error("unrecognized `algorithms' subcommand")
return 1
@algorithms.command()
@click.argument('name', nargs=-1)
@click.option('--force', help='Performs operation regardless of conflicts',
is_flag=True)
@click.option('--dry-run', help="Doesn't really perform the task, just "
"comments what would do", is_flag=True)
@click.pass_context
def push(ctx, name, force, dry_run):
'''Uploads algorithms to the server
Example:
$ beat algorithms push --dry-run yyy
'''
with common.make_webapi(ctx.meta['config']) as webapi:
return common.push(webapi, ctx.meta['config'].path, 'algorithm',
name, ['name', 'declaration', 'code', 'description'],
{}, force, dry_run, 0)
@algorithms.command()
@click.argument('name', nargs=1)
@click.pass_context
def diff(ctx, name):
'''Shows changes between the local algorithm and the remote version
Example:
$ beat algorithms diff xxx
'''
with common.make_webapi(ctx.meta['config']) as webapi:
return common.diff(webapi, ctx.meta['config'].path, 'algorithm',
name, ['declaration', 'code', 'description'])
@algorithms.command()
@click.pass_context
def status(ctx):
'''Shows (editing) status for all available algorithms
Example:
$ beat algorithms status
'''
with common.make_webapi(ctx.meta['config']) as webapi:
return common.status(webapi, ctx.meta['config'].path, 'algorithm')[0]
@algorithms.command()
@click.argument('name', nargs=-1)
@click.pass_context
def create(ctx, name):
'''Creates a new local algorithm
Example:
$ beat algorithms create xxx
'''
return common.create(ctx.meta['config'].path, 'algorithm', name)
@algorithms.command()
@click.argument('name', nargs=1)
@click.pass_context
def version(ctx, name):
'''Creates a new version of an existing algorithm
Example:
$ beat algorithms version xxx
'''
return common.new_version(ctx.meta['config'].path, 'algorithm', name)
@algorithms.command()
@click.argument('src', nargs=1)
@click.argument('dst', nargs=1)
@click.pass_context
def fork(ctx, src, dst):
'''Forks a local algorithm
Example:
$ beat algorithms fork xxx yyy
'''
return common.fork(ctx.meta['config'].path, 'algorithm', src, dst)
@algorithms.command()
@click.argument('name', nargs=-1)
@click.option('--remote', help='Only acts on the remote copy of the algorithm',
is_flag=True)
@click.pass_context
def rm(ctx, name, remote):
'''Deletes a local algorithm (unless --remote is specified)
Example:
$ beat algorithms rm xxx
'''
if remote:
with common.make_webapi(ctx.meta['config']) as webapi:
return common.delete_remote(webapi, 'algorithm', name)
else:
return common.delete_local(ctx.meta['config'].path, 'algorithm', name)
@algorithms.command()
@click.argument('instructions', nargs=1)
@click.option('--example', help='Display some example JSON instruction files',
is_flag=True)
@click.pass_context
def execute(ctx, instructions, example):
'''Execute an algorithm following instructions in a JSON file
Example:
$ beat algorithms execute <instructions>
$ beat algorithms execute --examples
'''
if example:
print_examples()
return 0
return execute_impl(ctx.meta['config'].path, ctx.meta['config'].cache,
instructions)
......@@ -25,46 +25,8 @@
# #
###############################################################################
"""Configuration manipulation and display
Usage:
%(prog)s cache clear
%(prog)s cache info [--start=<index>] [--end=<index>] [--sizes] [<path>]...
%(prog)s cache view [--start=<index>] [--end=<index>] [<path>]...
%(prog)s cache --help
Commands:
clear Deletes all available cache
info Displays information about a particular cache file
view Displays the contents of a particular cache file
Options:
-s, --sizes If set, also print the size in bytes for objects in a file. This
triggers the full file readout
--start=<i> If set, allows the user to print only a few bits of the file
--end=<i> If set, allows the user to print only a few bits of the file
-h, --help Display this screen
Examples:
To clear all available cache:
$ %(prog)s cache clear
To collect information about a particular cache file:
$ %(prog)s cache info 7f/d8/8d/a11178ac27075feaba8131fe878d6e3...
To view a particular cache file:
$ %(prog)s cache view 7f/d8/8d/a11178ac27075feaba8131fe878d6e3...
"""
import os
import click
import fnmatch
import logging
logger = logging.getLogger(__name__)
......@@ -72,124 +34,150 @@ logger = logging.getLogger(__name__)
import simplejson
from . import common
from beat.cmdline.scripts.click_helper import AliasedGroup
from beat.core.data import CachedDataSource, load_data_index
from beat.core.utils import NumpyJSONEncoder
def get_paths(config):
func = lambda z: z.split('.', 1)[0]
retval = []
for dirname, dirs, files in os.walk(config.cache):
files = fnmatch.filter(files, '*.data') #avoid index-only files
if not files: continue
d = dirname.replace(config.cache, '').strip(os.sep)
retval += list(set([os.path.join(d, func(k)) for k in files]))
func = lambda z: z.split('.', 1)[0]
return retval
retval = []
for dirname, _, files in os.walk(config.cache):
files = fnmatch.filter(files, '*.data') #avoid index-only files
if not files:
continue
d = dirname.replace(config.cache, '').strip(os.sep)
retval += list(set([os.path.join(d, func(k)) for k in files]))
def info(config, paths, sizes, index_start, index_end):
return retval
if not paths:
paths = get_paths(config)
for path in paths:
@click.group(cls=AliasedGroup)
@click.pass_context
@click.option('--start', type=click.INT, help='If set, allows the user to '
'print only a few bits of the file')
@click.option('--end', type=click.INT, help='If set, allows the user to '
'print only a few bits of the file')
def cache(ctx, start, end):
"""Configuration manipulation and display"""
pass
logger.info('path: %s', path)
fullpath = os.path.join(config.cache, path + '.data')
f = CachedDataSource()
status = f.setup(fullpath, config.path, index_start, index_end)
if not status:
logger.error("cannot setup data source with `%s' and prefix `%s'",
fullpath, config.path)
return 1
@cache.command()
@click.pass_context
def clear(ctx):
'''Deletes all available cache
logger.info(' dataformat: %s', f.dataformat.name)
To clear all available cache:
if sizes:
counter = 0
logger.info(' index:')
for data, start, end in f:
size = len(data.pack())
counter += size
if start == end:
logger.info(' [%d] - %d bytes', start, size)
else:
logger.info(' [%d:%d] - %d bytes', start, end, size)
logger.info(' total (stripped-down) size: %d bytes', counter)
else:
index = load_data_index(config.cache, path + '.data')
logger.info(' objects : %d', len(index)-1)
return 0
def view(config, paths, index_start, index_end):
if not paths:
paths = get_paths(config)
for path in paths:
logger.info('path: %s', path)
fullpath = os.path.join(config.cache, path + '.data')
f = CachedDataSource()
status = f.setup(fullpath, config.path, index_start, index_end)
if not status:
logger.error("cannot setup data source with `%s' and prefix `%s'",
fullpath, config.path)
return 1
logger.info(' dataformat: %s', f.dataformat.name)
for data, start, end in f:
logger.extra(80 * '-')
if start == end:
header = '[%d]: ' % start
else:
header = '[%d:%d]: ' % (start, end)
$ %(prog)s cache clear
'''
import shutil
if os.path.isdir(ctx.meta['config'].cache):
for k in os.listdir(ctx.meta['config'].cache):
p = os.path.join(ctx.meta['config'].cache, k)
shutil.rmtree(p)
json_data = data.as_dict()
for name, value in json_data.items():
json_data[name] = common.stringify(value)
json_data = simplejson.dumps(json_data, indent=2,
cls=NumpyJSONEncoder).\
replace('"BEAT_LIST_DELIMITER[', '[')\
.replace(']BEAT_LIST_DELIMITER"', ']')\
.replace('"...",', '...')\
.replace('"BEAT_LIST_SIZE(', '(')\
.replace(')BEAT_LIST_SIZE"', ')')
logger.info(header + json_data)
return 0
@cache.command()
@click.argument('paths', nargs=-1, type=click.Path(exists=True))
@click.pass_context
@click.option('--sizes', help='If set, also print the size in bytes for '
'objects in a file. This triggers the full file readout',
is_flag=True)
def info(ctx, paths, sizes):
'''Displays information about a particular cache file
To collect information about a particular cache file:
def process(args):
$ %(prog)s cache info 7f/d8/8d/a11178ac27075feaba8131fe878d6e3...
'''
config = ctx.meta['config']
index_start = int(ctx.meta['start']) if 'start' in ctx.meta else None
index_end = int(ctx.meta['end']) if 'end' in ctx.meta else None
if not paths:
paths = get_paths(config)
for path in paths:
logger.info('path: %s', path)
fullpath = os.path.join(config.cache, path + '.data')
f = CachedDataSource()
status = f.setup(fullpath, config.path, index_start, index_end)
if not status:
logger.error("cannot setup data source with `%s' and prefix `%s'",
fullpath, config.path)
return 1
logger.info(' dataformat: %s', f.dataformat.name)
if sizes:
counter = 0
logger.info(' index:')
for data, start, end in f:
size = len(data.pack())
counter += size
if start == end:
logger.info(' [%d] - %d bytes', start, size)
else:
logger.info(' [%d:%d] - %d bytes', start, end, size)
logger.info(' total (stripped-down) size: %d bytes', counter)
start = int(args['--start']) if args['--start'] is not None else None
end = int(args['--end']) if args['--end'] is not None else None
else:
index = load_data_index(config.cache, path + '.data')
logger.info(' objects : %d', len(index)-1)
if args['clear']:
import shutil
for k in os.listdir(args['config'].cache):
p = os.path.join(args['config'].cache, k)
shutil.rmtree(p)
return 0