Commit ae8dee63 authored by André Anjos's avatar André Anjos 💬
Browse files

Merge branch 'implement_subprocess_different_execution_environment' into 'master'

Implement subprocess different execution environment

See merge request !75
parents 96a977ac 0a53e506
Pipeline #34390 passed with stages
in 7 minutes and 59 seconds
......@@ -37,15 +37,45 @@ import click
class AliasedGroup(click.Group):
''' Class that handles prefix aliasing for commands '''
""" Class that handles prefix aliasing for commands """
def get_command(self, ctx, cmd_name):
rv = click.Group.get_command(self, ctx, cmd_name)
if rv is not None:
return rv
matches = [x for x in self.list_commands(ctx)
if x.startswith(cmd_name)]
matches = [x for x in self.list_commands(ctx) if x.startswith(cmd_name)]
if not matches:
return None
elif len(matches) == 1:
return click.Group.get_command(self, ctx, matches[0])
ctx.fail('Too many matches: %s' % ', '.join(sorted(matches)))
ctx.fail("Too many matches: %s" % ", ".join(sorted(matches)))
class MutuallyExclusiveOption(click.Option):
"""Class implementing mutually exclusive option
From https://stackoverflow.com/a/37491504/5843716
"""
def __init__(self, *args, **kwargs):
self.mutually_exclusive = set(kwargs.pop("mutually_exclusive", []))
help_ = kwargs.get("help", "")
if self.mutually_exclusive:
ex_str = ", ".join(self.mutually_exclusive)
kwargs["help"] = (
"{}\n"
"NOTE: This argument is mutually exclusive with arguments: [{}].".format(
help_, ex_str
)
)
super().__init__(*args, **kwargs)
def handle_parse_result(self, ctx, opts, args):
if self.mutually_exclusive.intersection(opts) and self.name in opts:
raise click.UsageError(
"Illegal usage: `{}` is mutually exclusive with "
"arguments `{}`.".format(self.name, ", ".join(self.mutually_exclusive))
)
return super().handle_parse_result(ctx, opts, args)
......@@ -51,6 +51,7 @@ from datetime import timedelta
from beat.core.experiment import Experiment
from beat.core.execution import DockerExecutor
from beat.core.execution import LocalExecutor
from beat.core.execution import SubprocessExecutor
from beat.core.utils import NumpyJSONEncoder
from beat.core.data import CachedDataSource, load_data_index
from beat.core.dock import Host
......@@ -64,12 +65,15 @@ from .plotters import plot_impl as plotters_plot
from .plotters import pull_impl as plotters_pull
from .decorators import raise_on_error
from .click_helper import AliasedGroup
from .click_helper import MutuallyExclusiveOption
logger = logging.getLogger(__name__)
def run_experiment(configuration, name, force, use_docker, use_local, quiet):
def run_experiment(
configuration, name, force, use_docker, use_local, run_environment_path, quiet
):
"""Run experiments locally"""
def load_result(executor):
......@@ -209,25 +213,37 @@ def run_experiment(configuration, name, force, use_docker, use_local, quiet):
if use_docker:
executor = DockerExecutor(
host,
configuration.path,
value["configuration"],
configuration.cache,
dataformat_cache,
database_cache,
algorithm_cache,
library_cache,
host=host,
prefix=configuration.path,
data=value["configuration"],
cache=configuration.cache,
dataformat_cache=dataformat_cache,
database_cache=database_cache,
algorithm_cache=algorithm_cache,
library_cache=library_cache,
)
else:
elif use_local:
executor = LocalExecutor(
configuration.path,
value["configuration"],
configuration.cache,
dataformat_cache,
database_cache,
algorithm_cache,
library_cache,
configuration.database_paths,
prefix=configuration.path,
data=value["configuration"],
cache=configuration.cache,
dataformat_cache=dataformat_cache,
database_cache=database_cache,
algorithm_cache=algorithm_cache,
library_cache=library_cache,
custom_root_folders=configuration.database_paths,
)
else:
executor = SubprocessExecutor(
prefix=configuration.path,
data=value["configuration"],
cache=configuration.cache,
dataformat_cache=dataformat_cache,
database_cache=database_cache,
algorithm_cache=algorithm_cache,
library_cache=library_cache,
custom_root_folders=configuration.database_paths,
python_path=run_environment_path,
)
if not executor.valid:
......@@ -713,6 +729,8 @@ commands.initialise_asset_commands(experiments, CMD_LIST)
help="Uses the docker executor to execute the "
"experiment using docker containers",
is_flag=True,
cls=MutuallyExclusiveOption,
mutually_exclusive=["local", "environment"],
)
@click.option(
"--local",
......@@ -720,14 +738,25 @@ commands.initialise_asset_commands(experiments, CMD_LIST)
"experiment on the local machine (default)",
default=True,
is_flag=True,
cls=MutuallyExclusiveOption,
mutually_exclusive=["docker", "environment"],
)
@click.option(
"--environment",
help="Uses the local executor to execute the "
"experiment on the local machine using the given environment."
"Given path should be to the python executable of the the environment",
type=click.Path(exists=True),
cls=MutuallyExclusiveOption,
mutually_exclusive=["docker", "local"],
)
@click.option("--quiet", help="Be less verbose", is_flag=True)
@click.pass_context
@raise_on_error
def run(ctx, name, force, docker, local, quiet):
def run(ctx, name, force, docker, local, environment, quiet):
""" Runs an experiment locally"""
config = ctx.meta.get("config")
return run_experiment(config, name, force, docker, local, quiet)
return run_experiment(config, name, force, docker, local, environment, quiet)
@experiments.command()
......
......@@ -40,13 +40,18 @@ import os
import logging
import nose.tools
import click
from click.testing import CliRunner
from . import platform, disconnected, prefix, tmp_prefix, user, token
from .utils import index_experiment_dbs, MockLoggingHandler
from ..common import Selector
from beat.cmdline.scripts import main_cli
from beat.core.test.utils import slow, cleanup, skipif
from beat.core.experiment import Storage, Experiment
from beat.core.test.test_execution import create_conda_environment
from beat.core.test.test_execution import clear_conda_environment
def setup_experiments():
......@@ -60,28 +65,31 @@ def call(*args, **kwargs):
use_prefix = kwargs.get("prefix", prefix)
use_platform = kwargs.get("platform", platform)
use_cache = kwargs.get("cache", "cache")
environment = kwargs.pop("environment", None)
cli_args = [
"--test-mode",
"--prefix",
use_prefix,
"--token",
token,
"--user",
user,
"--platform",
use_platform,
"--cache",
use_cache,
"experiments",
]
cli_args += list(args)
if environment:
cli_args += ["--environment", environment]
runner = CliRunner()
with runner.isolated_filesystem():
result = runner.invoke(
main_cli.main,
[
"--test-mode",
"--prefix",
use_prefix,
"--token",
token,
"--user",
user,
"--platform",
use_platform,
"--cache",
use_cache,
"experiments",
]
+ list(args),
catch_exceptions=False,
)
result = runner.invoke(main_cli.main, cli_args, catch_exceptions=False)
if result.exit_code != 0:
click.echo(result.output)
return result.exit_code
......@@ -180,6 +188,24 @@ def test_run_integers_addition_1():
nose.tools.eq_(call("run", obj, cache=tmp_prefix), 0)
@slow
@nose.tools.with_setup(setup=setup_experiments, teardown=cleanup)
def test_run_integers_addition_1_different_environment():
environment_path = create_conda_environment(["beat.backend.python"])
obj = "user/user/integers_addition/1/integers_addition"
status = call(
"run",
obj,
cache=tmp_prefix,
environment=os.path.join(environment_path, "bin", "python"),
)
clear_conda_environment(environment_path)
nose.tools.eq_(status, 0)
@slow
@nose.tools.with_setup(setup=setup_experiments, teardown=cleanup)
def test_list_integers_addition_1_cache():
......
......@@ -42,7 +42,7 @@ requirements:
- python
- setuptools
- beat.backend.python
- beat.core
- beat.core >1.9.0
- termcolor
- docopt
- click
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment