Commit 236b115a authored by André Anjos's avatar André Anjos 💬

Merge branch 'smart_push' into 'master'

Smart push

Closes #49 and #50

See merge request !77
parents 4ef644a3 9f4af24f
Pipeline #34668 passed with stages
in 7 minutes and 27 seconds
......@@ -32,6 +32,7 @@ libraries/
toolchains/
plotters/
plotterparameters/
protocoltemplates/
.noseids
scripts/_core_docker_pull.sh
_ci/
......@@ -45,12 +45,15 @@ from beat.core.execution import DockerExecutor
from beat.core.dock import Host
from beat.core import hash
from beat.backend.python.algorithm import Storage as AlgorithmStorage
from beat.backend.python.algorithm import Algorithm
from . import common
from . import commands
from .decorators import raise_on_error
from .click_helper import AliasedGroup
from .click_helper import AssetCommand
from .click_helper import AssetInfo
logger = logging.getLogger(__name__)
......@@ -325,14 +328,37 @@ def execute_impl(prefix, cache, instructions_file):
return 0
def get_dependencies(ctx, asset_name):
prefix = ctx.meta["config"].path
alg = Algorithm(prefix, asset_name)
dependencies = {}
libraries = list(alg.libraries.keys())
if libraries:
dependencies["libraries"] = libraries
dataformats = list(alg.dataformats.keys())
if dataformats:
dependencies["dataformats"] = dataformats
return dependencies
class AlgorithmCommand(AssetCommand):
asset_info = AssetInfo(
asset_type="algorithm",
diff_fields=["declaration", "code", "description"],
push_fields=["name", "declaration", "code", "description"],
get_dependencies=get_dependencies,
)
@click.group(cls=AliasedGroup)
@click.pass_context
def algorithms(ctx):
"""Configuration and manipulation of algorithms"""
ctx.meta["asset_type"] = "algorithm"
ctx.meta["diff_fields"] = ["declaration", "code", "description"]
CMD_LIST = [
"list",
......@@ -345,9 +371,10 @@ CMD_LIST = [
"fork",
"rm",
"diff",
"push",
]
commands.initialise_asset_commands(algorithms, CMD_LIST)
commands.initialise_asset_commands(algorithms, CMD_LIST, AlgorithmCommand)
@algorithms.command()
......@@ -367,38 +394,6 @@ def pull(ctx, name, force):
return pull_impl(webapi, ctx.meta["config"].path, name, force, 0, {}, {})
@algorithms.command()
@click.argument("name", nargs=-1)
@click.option(
"--force", help="Performs operation regardless of conflicts", is_flag=True
)
@click.option(
"--dry-run",
help="Doesn't really perform the task, just " "comments what would do",
is_flag=True,
)
@click.pass_context
@raise_on_error
def push(ctx, name, force, dry_run):
"""Uploads algorithms to the server
Example:
$ beat algorithms push --dry-run yyy
"""
with common.make_webapi(ctx.meta["config"]) as webapi:
return common.push(
webapi,
ctx.meta["config"].path,
"algorithm",
name,
["name", "declaration", "code", "description"],
{},
force,
dry_run,
0,
)
@algorithms.command()
@click.argument("instructions", nargs=1)
@click.option(
......
......@@ -32,6 +32,7 @@
# #
###################################################################################
"""Click based helper classes"""
import click
......@@ -40,6 +41,8 @@ class AliasedGroup(click.Group):
""" Class that handles prefix aliasing for commands """
def get_command(self, ctx, cmd_name):
"""Re-imp"""
rv = click.Group.get_command(self, ctx, cmd_name)
if rv is not None:
return rv
......@@ -58,6 +61,8 @@ class MutuallyExclusiveOption(click.Option):
"""
def __init__(self, *args, **kwargs):
"""Initialize"""
self.mutually_exclusive = set(kwargs.pop("mutually_exclusive", []))
help_ = kwargs.get("help", "")
if self.mutually_exclusive:
......@@ -72,6 +77,8 @@ class MutuallyExclusiveOption(click.Option):
super().__init__(*args, **kwargs)
def handle_parse_result(self, ctx, opts, args):
"""Re-imp"""
if self.mutually_exclusive.intersection(opts) and self.name in opts:
raise click.UsageError(
"Illegal usage: `{}` is mutually exclusive with "
......@@ -79,3 +86,36 @@ class MutuallyExclusiveOption(click.Option):
)
return super().handle_parse_result(ctx, opts, args)
class AssetInfo:
""" Information needed by the command to properly call local and remote
commands
"""
def __init__(
self, asset_type=None, diff_fields=None, push_fields=None, get_dependencies=None
):
self.asset_type = asset_type
self.diff_fields = diff_fields
self.push_fields = push_fields
self.get_dependencies = get_dependencies
def __repr__(self):
return "{}\n{}\n{}\n{}".format(
self.asset_type, self.diff_fields, self.push_fields, self.get_dependencies
)
class AssetCommand(click.Command):
""" Custom click command that will update the context with asset information
related to the command called.
"""
asset_info = AssetInfo()
def invoke(self, ctx):
"""Re-imp"""
ctx.meta["asset_info"] = self.asset_info
return super().invoke(ctx)
......@@ -36,6 +36,7 @@
import click
import types
from .scripts import main_cli
from .decorators import raise_on_error
from . import common
......@@ -71,13 +72,14 @@ def list_impl(ctx, remote):
$ beat <asset_type> list
"""
config = ctx.meta["config"]
asset_info = ctx.meta["asset_info"]
if remote:
with common.make_webapi(ctx.meta["config"]) as webapi:
return common.display_remote_list(webapi, ctx.meta["asset_type"])
with common.make_webapi(config) as webapi:
return common.display_remote_list(webapi, asset_info.asset_type)
else:
return common.display_local_list(
ctx.meta["config"].path, ctx.meta["asset_type"]
)
return common.display_local_list(config.path, asset_info.asset_type)
@click.argument("names", nargs=-1)
......@@ -90,9 +92,10 @@ def path_impl(ctx, names):
$ beat <asset_type> path xxx
"""
return common.display_local_path(
ctx.meta["config"].path, ctx.meta["asset_type"], names
)
config = ctx.meta["config"]
asset_info = ctx.meta["asset_info"]
return common.display_local_path(config.path, asset_info.asset_type, names)
@click.argument("name", nargs=1)
......@@ -104,9 +107,11 @@ def edit_impl(ctx, name):
Example:
$ beat <asset_type> edit xxx
"""
config = ctx.meta["config"]
asset_info = ctx.meta["asset_info"]
return common.edit_local_file(
ctx.meta["config"].path, ctx.meta["config"].editor, ctx.meta["asset_type"], name
config.path, config.editor, asset_info.asset_type, name
)
......@@ -119,8 +124,10 @@ def check_impl(ctx, names):
Example:
$ beat <asset_type> check xxx
"""
config = ctx.meta["config"]
asset_info = ctx.meta["asset_info"]
return common.check(ctx.meta["config"].path, ctx.meta["asset_type"], names)
return common.check(config.path, asset_info.asset_type, names)
@click.pass_context
......@@ -133,8 +140,10 @@ def status_impl(ctx):
"""
config = ctx.meta["config"]
asset_info = ctx.meta["asset_info"]
with common.make_webapi(config) as webapi:
return common.status(webapi, config.path, ctx.meta["asset_type"])[0]
return common.status(webapi, config.path, asset_info.asset_type)[0]
@click.argument("names", nargs=-1)
......@@ -147,7 +156,10 @@ def create_impl(ctx, names):
$ beat <asset_type> create xxx
"""
return common.create(ctx.meta["config"].path, ctx.meta["asset_type"], names)
config = ctx.meta["config"]
asset_info = ctx.meta["asset_info"]
return common.create(config.path, asset_info.asset_type, names)
@click.argument("name", nargs=1)
......@@ -160,7 +172,10 @@ def version_impl(ctx, name):
$ beat <asset_type> version xxx
"""
return common.new_version(ctx.meta["config"].path, ctx.meta["asset_type"], name)
config = ctx.meta["config"]
asset_info = ctx.meta["asset_info"]
return common.new_version(config.path, asset_info.asset_type, name)
@click.argument("src", nargs=1)
......@@ -174,7 +189,10 @@ def fork_impl(ctx, src, dst):
$ beat <asset_type> fork xxx yyy
"""
return common.fork(ctx.meta["config"].path, ctx.meta["asset_type"], src, dst)
config = ctx.meta["config"]
asset_info = ctx.meta["asset_info"]
return common.fork(config.path, asset_info.asset_type, src, dst)
@click.argument("name", nargs=-1)
......@@ -191,13 +209,13 @@ def rm_impl(ctx, name, remote):
"""
config = ctx.meta["config"]
asset_type = ctx.meta["asset_type"]
asset_info = ctx.meta["asset_info"]
if remote:
with common.make_webapi(config) as webapi:
return common.delete_remote(webapi, asset_type, name)
return common.delete_remote(webapi, asset_info.asset_type, name)
else:
return common.delete_local(config.path, asset_type, name)
return common.delete_local(config.path, asset_info.asset_type, name)
@click.argument("name", nargs=-1)
......@@ -209,25 +227,92 @@ def rm_local_impl(ctx, name):
Example:
$ beat <asset_type> rm xxx
"""
config = ctx.meta["config"]
asset_info = ctx.meta["asset_info"]
return common.delete_local(ctx.meta["config"].path, ctx.meta["asset_type"], name)
return common.delete_local(config.path, asset_info.asset_type, name)
@click.argument("name", nargs=1)
@click.pass_context
@raise_on_error
def diff_impl(ctx, name):
"""Shows changes between the local dataformat and the remote version
"""Shows changes between the local asset and the remote version
Example:
$ beat toolchains diff xxx
"""
config = ctx.meta["config"]
asset_info = ctx.meta["asset_info"]
with common.make_webapi(config) as webapi:
return common.diff(
webapi, config.path, ctx.meta["asset_type"], name, ctx.meta["diff_fields"]
webapi, config.path, asset_info.asset_type, name, asset_info.diff_fields
)
@click.argument("names", nargs=-1)
@click.option(
"--force", help="Performs operation regardless of conflicts", is_flag=True
)
@click.option(
"--dry-run",
help="Doesn't really perform the task, just " "comments what would do",
is_flag=True,
)
@click.pass_context
@raise_on_error
def push_impl(ctx, names, force, dry_run):
"""Uploads asset to the server
Example:
$ beat algorithms push --dry-run yyy
"""
def do_push(ctx, dependency_type, names):
push_cmd = main_cli.main.get_command(ctx, dependency_type).get_command(
ctx, "push"
)
ctx.meta["asset_info"] = push_cmd.asset_info
return ctx.invoke(push_cmd, names=names, force=force, dry_run=dry_run)
config = ctx.meta["config"]
asset_info = ctx.meta["asset_info"]
mappings = ctx.meta.get("mappings", {})
for name in names:
with common.Selector(config.path) as selector:
dependency_type = common.TYPE_PLURAL[asset_info.asset_type]
fork = selector.forked_from(asset_info.asset_type, name)
if fork:
status = do_push(ctx, dependency_type, [fork])
if status != 0:
return status
version = selector.version_of(asset_info.asset_type, name)
if version:
status = do_push(ctx, dependency_type, [version])
if status != 0:
return status
if asset_info.get_dependencies is not None:
dependencies = asset_info.get_dependencies(ctx, name)
for dependency_type, dependency_list in dependencies.items():
status = do_push(ctx, dependency_type, dependency_list)
if status != 0:
return status
with common.make_webapi(config) as webapi:
return common.push(
webapi=webapi,
prefix=config.path,
asset_type=asset_info.asset_type,
names=names,
fields=asset_info.push_fields,
mappings=mappings,
force=force,
dry_run=dry_run,
indentation=0,
)
......@@ -243,6 +328,7 @@ CMD_TABLE = {
"rm": rm_impl,
"rm_local": rm_local_impl,
"diff": diff_impl,
"push": push_impl,
}
......@@ -259,16 +345,17 @@ def command(name):
return copy_func(CMD_TABLE[name])
def initialise_asset_commands(click_cmd_group, cmd_list):
def initialise_asset_commands(click_cmd_group, cmd_list, cmd_cls):
"""Initialize a command group adding all the commands from cmd_list to it
Parameters:
click_cmd_group obj: click command to group
cmd_list list: list of string or tuple of the commands to add
cmd_cls: subclass of click Command to use
"""
for item in cmd_list:
if isinstance(item, tuple):
click_cmd_group.command(name=item[0])(command(item[1]))
click_cmd_group.command(cls=cmd_cls, name=item[0])(command(item[1]))
else:
click_cmd_group.command(name=item)(command(item))
click_cmd_group.command(cls=cmd_cls, name=item)(command(item))
This diff is collapsed.
......@@ -56,6 +56,8 @@ from . import commands
from .decorators import raise_on_error
from .click_helper import AliasedGroup
from .click_helper import AssetCommand
from .click_helper import AssetInfo
logger = logging.getLogger(__name__)
......@@ -649,14 +651,19 @@ def view_outputs(
# ----------------------------------------------------------
class DatabaseCommand(AssetCommand):
asset_info = AssetInfo(
asset_type="database",
diff_fields=["declaration", "code", "description"],
push_fields=["name", "declaration", "code", "description"],
)
@click.group(cls=AliasedGroup)
@click.pass_context
def databases(ctx):
"""Database commands"""
ctx.meta["asset_type"] = "database"
ctx.meta["diff_fields"] = ["declaration", "code", "description"]
CMD_LIST = [
"list",
......@@ -668,9 +675,10 @@ CMD_LIST = [
"version",
("rm", "rm_local"),
"diff",
"push",
]
commands.initialise_asset_commands(databases, CMD_LIST)
commands.initialise_asset_commands(databases, CMD_LIST, DatabaseCommand)
@databases.command()
......@@ -693,38 +701,6 @@ def pull(ctx, db_names, force):
return pull_impl(webapi, configuration.path, db_names, force, 0, {})
@databases.command()
@click.argument("db_names", nargs=-1)
@click.option(
"--force", help="Performs operation regardless of conflicts", is_flag=True
)
@click.option("--dry-run", help="Dry run", is_flag=True)
@click.pass_context
@raise_on_error
def push(ctx, db_names, force, dry_run):
"""Uploads databases to the server (must provide a valid admin token).
$ beat databases push [<name>]...
<name>:
Database name formatted as "<database>/<version>"
"""
configuration = ctx.meta["config"]
with common.make_webapi(configuration) as webapi:
return common.push(
webapi,
configuration.path,
"database",
db_names,
["name", "declaration", "code", "description"],
{},
force,
dry_run,
0,
)
@databases.command()
@click.argument("db_names", nargs=-1)
@click.option(
......
......@@ -39,12 +39,15 @@ import click
from beat.core import dataformat
from beat.backend.python.dataformat import DataFormat
from . import common
from . import commands
from .decorators import raise_on_error
from .click_helper import AliasedGroup
from .click_helper import AssetCommand
from .click_helper import AssetInfo
logger = logging.getLogger(__name__)
......@@ -130,14 +133,33 @@ def pull_impl(webapi, prefix, names, force, indentation, cache):
return pull_impl(webapi, prefix, dataformats, force, 2 + indentation, cache)
def get_dependencies(ctx, asset_name):
prefix = ctx.meta["config"].path
df = DataFormat(prefix, asset_name)
dependencies = {}
dataformats = list(df.referenced.keys())
if dataformats:
dependencies["dataformats"] = dataformats
return dependencies
class DataformatCommand(AssetCommand):
asset_info = AssetInfo(
asset_type="dataformat",
diff_fields=["declaration", "description"],
push_fields=["name", "declaration", "description"],
get_dependencies=get_dependencies,
)
@click.group(cls=AliasedGroup)
@click.pass_context
def dataformats(ctx):
"""Configuration manipulation of data formats"""
ctx.meta["asset_type"] = "dataformat"
ctx.meta["diff_fields"] = ["declaration", "description"]
CMD_LIST = [
"list",
......@@ -150,9 +172,11 @@ CMD_LIST = [
"fork",
"rm",
"diff",
"push",
]
commands.initialise_asset_commands(dataformats, CMD_LIST)
commands.initialise_asset_commands(dataformats, CMD_LIST, DataformatCommand)
@dataformats.command()
......@@ -173,35 +197,3 @@ def pull(ctx, name, force):
if name is None:
return 1 # error
return pull_impl(webapi, ctx.meta["config"].path, name, force, 0, {})
@dataformats.command()
@click.argument("name", nargs=-1)
@click.option(
"--force", help="Performs operation regardless of conflicts", is_flag=True
)
@click.option(
"--dry-run",
help="Doesn't really perform the task, just " "comments what would do",
is_flag=True,
)
@click.pass_context
@raise_on_error
def push(ctx, name, force, dry_run):
"""Uploads dataformats to the server
Example:
$ beat dataformats push --dry-run yyy
"""
with common.make_webapi(ctx.meta["config"]) as webapi:
return common.push(
webapi,
ctx.meta["config"].path,
"dataformat",
name,
["name", "declaration", "description"],
{},
force,
dry_run,
0,
)
......@@ -40,7 +40,7 @@ from functools import wraps
from .log import set_verbosity_level
# This needs to be beat so that logger is configured for all beat packages.
logger = logging.getLogger('beat')
logger = logging.getLogger("beat")
def verbosity_option(**kwargs):
......@@ -56,20 +56,28 @@ def verbosity_option(**kwargs):
callable
A decorator to be used for adding this option.
"""
def custom_verbosity_option(f):
def callback(ctx, param, value):
ctx.meta['verbosity'] = value
ctx.meta["verbosity"] = value
set_verbosity_level(logger, value)
logger.debug("Logging of the `beat' logger was set to %d", value)
return value
return click.option(
'-v', '--verbose', count=True,
expose_value=False, default=2,
"-v",
"--verbose",
count=True,
expose_value=False,
default=2,
help="Increase the verbosity level from 0 (only error messages) "
"to 1 (warnings), 2 (log messages), 3 (debug information) by "
"adding the --verbose option as often as desired "
"(e.g. '-vvv' for debug).",
callback=callback, **kwargs)(f)
callback=callback,
**kwargs
)(f)
return custom_verbosity_option
......@@ -83,8 +91,11 @@ def raise_on_error(view_func):
def _decorator(*args, **kwargs):
value = view_func(*args, **kwargs)
if value not in [None, 0]:
exception = click.ClickException("Error occured")
exception = click.ClickException(
"Error occured: returned value is {}".format(value)
)
exception.exit_code = value
raise exception
return value
return wraps(view_func)(_decorator)
......@@ -36,7 +36,6 @@
import click
import logging
import six
import simplejson as json
from beat.core.dock import Host
......@@ -52,17 +51,7 @@ logger = logging.getLogger(__name__)
def get_remote_environments(ctx):
config = ctx.meta.get("config")
with common.make_webapi(config) as webapi:
status, answer = webapi.get("/api/v1/backend/environments/")
if status != six.moves.http_client.OK:
logger.error(
"failed to retrieve environment list on {}, reason: {}".format(
webapi.platform, six.moves.http_client.responses[status]
)
)
return {}
else:
return json.loads(answer)
return webapi.get("/api/v1/backend/environments/")
def get_docker_environments():
......
......@@ -63,9 +63,13 @@ from . import commands
from .plotters import plot_impl as plotters_plot
from .plotters import pull_impl as