Commit 324a3801 authored by Samuel GAIST's avatar Samuel GAIST

[algorithms] pre-commit cleanup

parent ada35613
......@@ -55,7 +55,7 @@ logger = logging.getLogger(__name__)
def pull_impl(webapi, prefix, names, force, indentation, format_cache, lib_cache):
"""Copies algorithms (and required libraries/dataformats) from the server.
"""Copies algorithms (and required libraries/dataformats) from the server.
Parameters:
......@@ -93,37 +93,48 @@ def pull_impl(webapi, prefix, names, force, indentation, format_cache, lib_cache
"""
from .dataformats import pull_impl as dataformats_pull
from .libraries import pull_impl as libraries_pull
from .dataformats import pull_impl as dataformats_pull
from .libraries import pull_impl as libraries_pull
status, names = common.pull(webapi, prefix, 'algorithm', names,
['declaration', 'code', 'description'], force, indentation)
status, names = common.pull(
webapi,
prefix,
"algorithm",
names,
["declaration", "code", "description"],
force,
indentation,
)
if status != 0: return status
if status != 0:
return status
# see what dataformats one needs to pull
indent = indentation * ' '
# see what dataformats one needs to pull
if indentation == 0:
indentation = 4
dataformats = []
libraries = []
for name in names:
obj = algorithm.Algorithm(prefix, name)
dataformats.extend(obj.dataformats.keys())
libraries.extend(obj.libraries.keys())
dataformats = []
libraries = []
for name in names:
obj = algorithm.Algorithm(prefix, name)
dataformats.extend(obj.dataformats.keys())
libraries.extend(obj.libraries.keys())
# downloads any formats to which we depend on
df_status = dataformats_pull(webapi, prefix, dataformats, force,
indentation + 2, format_cache)
# downloads any formats to which we depend on
df_status = dataformats_pull(
webapi, prefix, dataformats, force, indentation, format_cache
)
lib_status = libraries_pull(webapi, prefix, libraries, force,
indentation + 2, lib_cache)
return status + df_status + lib_status
lib_status = libraries_pull(
webapi, prefix, libraries, force, indentation, lib_cache
)
return status + df_status + lib_status
def print_examples():
print("""
print(
"""
To feed data from a database to an algorithm:
=============================================
......@@ -201,99 +212,115 @@ To execute an analyzer:
"version": "<environment_version>"
}
}
""")
"""
)
def execute_impl(prefix, cache, instructions_file):
try:
# Load the JSON configuration
if not os.path.exists(instructions_file):
raise IOError("JSON instructions file `%s' not found" % instructions_file)
with open(instructions_file, 'r') as f:
configuration = json.load(f)
# Add missing configuration fields
configuration['queue'] = 'unused'
configuration['nb_slots'] = 1
if 'parameters' not in configuration:
configuration['parameters'] = {}
for name, cfg in configuration['inputs'].items():
cfg['endpoint'] = name
suffix = ''
if 'database' in cfg: # Connected to a database output
cfg['hash'] = hash.hashDataset(cfg['database'], cfg['protocol'], cfg['set'])
suffix = '.db'
cfg['path'] = hash.toPath(cfg['hash'], suffix=suffix)
algo = AlgorithmStorage(prefix, configuration['algorithm'])
if 'outputs' in configuration: # Standard algorithm
for name, cfg in configuration['outputs'].items():
cfg['endpoint'] = name
cfg['hash'] = hash.hashBlockOutput(
'block',
configuration['algorithm'],
algo.hash(),
configuration['parameters'],
configuration['environment'],
dict([(k, v['hash']) for k, v in configuration['inputs'].items()]),
name,
)
cfg['path'] = hash.toPath(cfg['hash'], '')
else: # Analyzer
configuration['result'] = {}
configuration['result']['hash'] = hash.hashAnalyzer(
'block',
configuration['algorithm'],
algo.hash(),
configuration['parameters'],
configuration['environment'],
dict([(k, v['hash']) for k, v in configuration['inputs'].items()]),
)
configuration['result']['path'] = hash.toPath(configuration['result']['hash'], '')
# Sets up the execution
dataformat_cache = {}
database_cache = {}
algorithm_cache = {}
host = Host(raise_on_errors=False)
executor = DockerExecutor(host, prefix, configuration, cache, dataformat_cache,
database_cache, algorithm_cache)
if not executor.valid:
logger.error('Invalid configuration:\n * %s' % '\n * '.join(executor.errors))
return 1
# Execute the algorithm
with executor:
result = executor.process()
if result['status'] != 0:
print('STDERR:')
print(result['stderr'])
# Display the results
if 'outputs' in configuration: # Standard algorithm
print('Outputs of the algorithms available at:')
for name, cfg in configuration['outputs'].items():
print(' - %s: %s' % (name, cfg['path']))
else:
print('Results of the analyzer available at: %s' % configuration['result']['path'])
except Exception as e:
import traceback
logger.error(traceback.format_exc())
return 1
return 0
try:
# Load the JSON configuration
if not os.path.exists(instructions_file):
raise IOError("JSON instructions file `%s' not found" % instructions_file)
with open(instructions_file, "r") as f:
configuration = json.load(f)
# Add missing configuration fields
configuration["queue"] = "unused"
configuration["nb_slots"] = 1
if "parameters" not in configuration:
configuration["parameters"] = {}
for name, cfg in configuration["inputs"].items():
cfg["endpoint"] = name
suffix = ""
if "database" in cfg: # Connected to a database output
cfg["hash"] = hash.hashDataset(
cfg["database"], cfg["protocol"], cfg["set"]
)
suffix = ".db"
cfg["path"] = hash.toPath(cfg["hash"], suffix=suffix)
algo = AlgorithmStorage(prefix, configuration["algorithm"])
if "outputs" in configuration: # Standard algorithm
for name, cfg in configuration["outputs"].items():
cfg["endpoint"] = name
cfg["hash"] = hash.hashBlockOutput(
"block",
configuration["algorithm"],
algo.hash(),
configuration["parameters"],
configuration["environment"],
dict([(k, v["hash"]) for k, v in configuration["inputs"].items()]),
name,
)
cfg["path"] = hash.toPath(cfg["hash"], "")
else: # Analyzer
configuration["result"] = {}
configuration["result"]["hash"] = hash.hashAnalyzer(
"block",
configuration["algorithm"],
algo.hash(),
configuration["parameters"],
configuration["environment"],
dict([(k, v["hash"]) for k, v in configuration["inputs"].items()]),
)
configuration["result"]["path"] = hash.toPath(
configuration["result"]["hash"], ""
)
# Sets up the execution
dataformat_cache = {}
database_cache = {}
algorithm_cache = {}
host = Host(raise_on_errors=False)
executor = DockerExecutor(
host,
prefix,
configuration,
cache,
dataformat_cache,
database_cache,
algorithm_cache,
)
if not executor.valid:
logger.error(
"Invalid configuration:\n * %s" % "\n * ".join(executor.errors)
)
return 1
# Execute the algorithm
with executor:
result = executor.process()
if result["status"] != 0:
print("STDERR:")
print(result["stderr"])
# Display the results
if "outputs" in configuration: # Standard algorithm
print("Outputs of the algorithms available at:")
for name, cfg in configuration["outputs"].items():
print(" - %s: %s" % (name, cfg["path"]))
else:
print(
"Results of the analyzer available at: %s"
% configuration["result"]["path"]
)
except Exception:
import traceback
logger.error(traceback.format_exc())
return 1
return 0
@click.group(cls=AliasedGroup)
......@@ -302,213 +329,225 @@ def algorithms(ctx):
"""Configuration and manipulation of algorithms"""
pass
@algorithms.command()
@click.option('--remote', help='Only acts on the remote copy of the algorithm',
is_flag=True)
@click.option(
"--remote", help="Only acts on the remote copy of the algorithm", is_flag=True
)
@click.pass_context
@raise_on_error
def list(ctx, remote):
'''Lists all the algorithms available on the platform
"""Lists all the algorithms available on the platform
Example:
$ beat algorithms list --remote
'''
if remote:
with common.make_webapi(ctx.meta['config']) as webapi:
return common.display_remote_list(webapi, 'algorithm')
else:
return common.display_local_list(ctx.meta['config'].path, 'algorithm')
"""
if remote:
with common.make_webapi(ctx.meta["config"]) as webapi:
return common.display_remote_list(webapi, "algorithm")
else:
return common.display_local_list(ctx.meta["config"].path, "algorithm")
@algorithms.command()
@click.argument('names', nargs=-1)
@click.argument("names", nargs=-1)
@click.pass_context
@raise_on_error
def path(ctx, names):
'''Displays local path of algorithm files
"""Displays local path of algorithm files
Example:
$ beat algorithms path xxx
'''
return common.display_local_path(ctx.meta['config'].path, 'algorithm', names)
"""
return common.display_local_path(ctx.meta["config"].path, "algorithm", names)
@algorithms.command()
@click.argument('name', nargs=1)
@click.argument("name", nargs=1)
@click.pass_context
@raise_on_error
def edit(ctx, name):
'''Edit local algorithm file
"""Edit local algorithm file
Example:
$ beat algorithms edit xxx
'''
return common.edit_local_file(ctx.meta['config'].path,
ctx.meta['config'].editor, 'algorithm',
name)
"""
return common.edit_local_file(
ctx.meta["config"].path, ctx.meta["config"].editor, "algorithm", name
)
@algorithms.command()
@click.argument('name', nargs=-1)
@click.argument("name", nargs=-1)
@click.pass_context
@raise_on_error
def check(ctx, name):
'''Checks a local algorithm for validity
"""Checks a local algorithm for validity
Example:
$ beat algorithms check xxx
'''
return common.check(ctx.meta['config'].path, 'algorithm', name)
"""
return common.check(ctx.meta["config"].path, "algorithm", name)
@algorithms.command()
@click.argument('name', nargs=-1)
@click.option('--force', help='Performs operation regardless of conflicts',
is_flag=True)
@click.argument("name", nargs=-1)
@click.option(
"--force", help="Performs operation regardless of conflicts", is_flag=True
)
@click.pass_context
@raise_on_error
def pull(ctx, name, force):
'''Downloads the specified algorithms from the server
"""Downloads the specified algorithms from the server
Example:
$ beat algorithms pull --force yyy
'''
with common.make_webapi(ctx.meta['config']) as webapi:
return pull_impl(webapi, ctx.meta['config'].path, name, force, 0, {}, {})
"""
with common.make_webapi(ctx.meta["config"]) as webapi:
return pull_impl(webapi, ctx.meta["config"].path, name, force, 0, {}, {})
@algorithms.command()
@click.argument('name', nargs=-1)
@click.option('--force', help='Performs operation regardless of conflicts',
is_flag=True)
@click.option('--dry-run', help="Doesn't really perform the task, just "
"comments what would do", is_flag=True)
@click.argument("name", nargs=-1)
@click.option(
"--force", help="Performs operation regardless of conflicts", is_flag=True
)
@click.option(
"--dry-run",
help="Doesn't really perform the task, just " "comments what would do",
is_flag=True,
)
@click.pass_context
@raise_on_error
def push(ctx, name, force, dry_run):
'''Uploads algorithms to the server
"""Uploads algorithms to the server
Example:
$ beat algorithms push --dry-run yyy
'''
with common.make_webapi(ctx.meta['config']) as webapi:
return common.push(webapi, ctx.meta['config'].path, 'algorithm',
name, ['name', 'declaration', 'code', 'description'],
{}, force, dry_run, 0)
"""
with common.make_webapi(ctx.meta["config"]) as webapi:
return common.push(
webapi,
ctx.meta["config"].path,
"algorithm",
name,
["name", "declaration", "code", "description"],
{},
force,
dry_run,
0,
)
@algorithms.command()
@click.argument('name', nargs=1)
@click.argument("name", nargs=1)
@click.pass_context
@raise_on_error
def diff(ctx, name):
'''Shows changes between the local algorithm and the remote version
"""Shows changes between the local algorithm and the remote version
Example:
$ beat algorithms diff xxx
'''
with common.make_webapi(ctx.meta['config']) as webapi:
return common.diff(webapi, ctx.meta['config'].path, 'algorithm',
name, ['declaration', 'code', 'description'])
"""
with common.make_webapi(ctx.meta["config"]) as webapi:
return common.diff(
webapi,
ctx.meta["config"].path,
"algorithm",
name,
["declaration", "code", "description"],
)
@algorithms.command()
@click.pass_context
@raise_on_error
def status(ctx):
'''Shows (editing) status for all available algorithms
"""Shows (editing) status for all available algorithms
Example:
$ beat algorithms status
'''
with common.make_webapi(ctx.meta['config']) as webapi:
return common.status(webapi, ctx.meta['config'].path, 'algorithm')[0]
"""
with common.make_webapi(ctx.meta["config"]) as webapi:
return common.status(webapi, ctx.meta["config"].path, "algorithm")[0]
@algorithms.command()
@click.argument('name', nargs=-1)
@click.argument("name", nargs=-1)
@click.pass_context
@raise_on_error
def create(ctx, name):
'''Creates a new local algorithm
"""Creates a new local algorithm
Example:
$ beat algorithms create xxx
'''
return common.create(ctx.meta['config'].path, 'algorithm', name)
"""
return common.create(ctx.meta["config"].path, "algorithm", name)
@algorithms.command()
@click.argument('name', nargs=1)
@click.argument("name", nargs=1)
@click.pass_context
@raise_on_error
def version(ctx, name):
'''Creates a new version of an existing algorithm
"""Creates a new version of an existing algorithm
Example:
$ beat algorithms version xxx
'''
return common.new_version(ctx.meta['config'].path, 'algorithm', name)
"""
return common.new_version(ctx.meta["config"].path, "algorithm", name)
@algorithms.command()
@click.argument('src', nargs=1)
@click.argument('dst', nargs=1)
@click.argument("src", nargs=1)
@click.argument("dst", nargs=1)
@click.pass_context
@raise_on_error
def fork(ctx, src, dst):
'''Forks a local algorithm
"""Forks a local algorithm
Example:
$ beat algorithms fork xxx yyy
'''
return common.fork(ctx.meta['config'].path, 'algorithm', src, dst)
"""
return common.fork(ctx.meta["config"].path, "algorithm", src, dst)
@algorithms.command()
@click.argument('name', nargs=-1)
@click.option('--remote', help='Only acts on the remote copy of the algorithm',
is_flag=True)
@click.argument("name", nargs=-1)
@click.option(
"--remote", help="Only acts on the remote copy of the algorithm", is_flag=True
)
@click.pass_context
@raise_on_error
def rm(ctx, name, remote):
'''Deletes a local algorithm (unless --remote is specified)
"""Deletes a local algorithm (unless --remote is specified)
Example:
$ beat algorithms rm xxx
'''
if remote:
with common.make_webapi(ctx.meta['config']) as webapi:
return common.delete_remote(webapi, 'algorithm', name)
else:
return common.delete_local(ctx.meta['config'].path, 'algorithm', name)
"""
if remote:
with common.make_webapi(ctx.meta["config"]) as webapi:
return common.delete_remote(webapi, "algorithm", name)
else:
return common.delete_local(ctx.meta["config"].path, "algorithm", name)
@algorithms.command()
@click.argument('instructions', nargs=1)
@click.option('--example', help='Display some example JSON instruction files',
is_flag=True)
@click.argument("instructions", nargs=1)
@click.option(
"--example", help="Display some example JSON instruction files", is_flag=True
)
@click.pass_context
@raise_on_error
def execute(ctx, instructions, example):
'''Execute an algorithm following instructions in a JSON file
"""Execute an algorithm following instructions in a JSON file
Example:
$ beat algorithms execute <instructions>
$ beat algorithms execute --examples
'''
if example:
print_examples()
return 0
"""
if example:
print_examples()
return 0
return execute_impl(ctx.meta['config'].path, ctx.meta['config'].cache,
instructions)
return execute_impl(ctx.meta["config"].path, ctx.meta["config"].cache, instructions)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment