Commit 78acb9d2 authored by Samuel GAIST's avatar Samuel GAIST Committed by Samuel GAIST

[experiments] Add option to share the database with the algorithm container

This allows to use the new feature implemented
for beat.core#103.
parent cbfb5f6f
Pipeline #45312 passed with stage
in 5 minutes and 2 seconds
......@@ -75,7 +75,14 @@ logger = logging.getLogger(__name__)
def run_experiment(
configuration, name, force, use_docker, use_local, run_environment_path, quiet
configuration,
name,
force,
use_docker,
use_local,
run_environment_path,
quiet,
share_databases,
):
"""Run experiments locally"""
......@@ -168,7 +175,7 @@ def run_experiment(
database_cache = {}
algorithm_cache = {}
library_cache = {}
# from .test.utils import set_trace; set_trace()
experiment = Experiment(
configuration.path,
name,
......@@ -227,10 +234,12 @@ def run_experiment(
python_path=run_environment_path,
)
elif use_docker:
exp_configuration = value["configuration"]
exp_configuration["share_databases"] = share_databases
executor = DockerExecutor(
host=host,
prefix=configuration.path,
data=value["configuration"],
data=exp_configuration,
cache=configuration.cache,
dataformat_cache=dataformat_cache,
database_cache=database_cache,
......@@ -774,12 +783,20 @@ commands.initialise_asset_commands(experiments, CMD_LIST, ExperimentCommand)
mutually_exclusive=["docker", "local"],
)
@click.option("--quiet", help="Be less verbose", is_flag=True)
@click.option(
"--share-databases",
help="Share the database files with the algorithm container",
is_flag=True,
default=False,
)
@click.pass_context
@raise_on_error
def run(ctx, name, force, docker, local, environment, quiet):
def run(ctx, name, force, docker, local, environment, quiet, share_databases):
""" Runs an experiment locally"""
config = ctx.meta.get("config")
return run_experiment(config, name, force, docker, local, environment, quiet)
return run_experiment(
config, name, force, docker, local, environment, quiet, share_databases
)
@experiments.command()
......
......@@ -39,13 +39,14 @@
import contextlib
import os
import shutil
import subprocess # nosec
import sys
import tempfile
import urllib
import pkg_resources
from beat.core.test import initialize_db_root_folder
from beat.core.test import sync_prefixes
from beat.core.test import teardown_package as bc_teardown_package
from beat.core.test import tmp_prefix # noqa forward import
......@@ -108,13 +109,18 @@ prefix = os.path.join(prefix_folder, "prefix")
def setup_package():
prefixes = [
pkg_resources.resource_filename("beat.backend.python.test", "prefix"),
pkg_resources.resource_filename("beat.core.test", "prefix"),
pkg_resources.resource_filename("beat.cmdline.test", "prefix"),
pkg_resources.resource_filename(f"beat.{resource}.test", "prefix")
for resource in ["backend.python", "core", "cmdline"]
]
for path in prefixes:
subprocess.check_call(["rsync", "-arz", path, prefix_folder]) # nosec
sync_prefixes(
prefixes, prefix_folder,
)
initialize_db_root_folder(
os.path.join(prefix_folder, "beat_cmdline_test"),
os.path.join(prefix, "databases"),
)
def teardown_package():
......
......@@ -227,3 +227,30 @@ class TestDockerExperimentRun(core.AssetBaseTest):
nose.tools.eq_(exit_code, 1, outputs)
exit_code, outputs = self.call("run", obj, "--docker", cache=tmp_prefix)
nose.tools.eq_(exit_code, 1, outputs)
def run_database_sharing(self, share_databases, expected_output):
"""Test that the database sharing works"""
db = Database(prefix, "integers_db/1")
nose.tools.assert_true(db.valid, db.errors)
data_sharing_path = db.data["root_folder"]
offset = 12
with open(os.path.join(data_sharing_path, "datafile.txt"), "wt") as data_file:
data_file.write("{}".format(offset))
obj = "user/user/integers_addition/1/shared_datasets"
args = ["run", obj, "--docker"]
if share_databases:
args.append("--share-databases")
exit_code, outputs = self.call(*args, cache=tmp_prefix)
nose.tools.eq_(exit_code, expected_output, outputs)
@slow
def test_run_database_sharing(self):
for share, result in [(False, 1), (True, 0)]:
yield self.run_database_sharing, share, result
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment