From 2e89d878d3336e5ba60e41b11c80c03d8206bda4 Mon Sep 17 00:00:00 2001 From: Samuel Gaist Date: Thu, 16 May 2019 15:12:50 +0200 Subject: [PATCH 1/3] [scripts][editor_cli] Add environment caching In order to avoid having the reload again and again the docker environment available, create a cache file if not existing on startup. This also add a command to manually refresh the cache if needed. --- beat/editor/scripts/editor_cli.py | 47 +++++++++++++++++++++++++++++-- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/beat/editor/scripts/editor_cli.py b/beat/editor/scripts/editor_cli.py index 26ad706..70dc5a1 100644 --- a/beat/editor/scripts/editor_cli.py +++ b/beat/editor/scripts/editor_cli.py @@ -39,6 +39,8 @@ from click_plugins import with_plugins from PyQt5.QtWidgets import QApplication from PyQt5.QtCore import QCoreApplication +from beat.core.dock import Host + from beat.cmdline.click_helper import AliasedGroup from beat.cmdline.decorators import raise_on_error from beat.cmdline.decorators import verbosity_option @@ -53,6 +55,13 @@ from .. import version global logger logger = None + +def dump_environments(environments_file_path): + logger.info("Generating environments information") + Host(images_cache=environments_file_path, raise_on_errors=False) + logger.info("Done") + + START_EPILOG = """\b Example: $ beat editor start @@ -62,18 +71,33 @@ Example: @with_plugins(pkg_resources.iter_entry_points("beat.editor.cli")) @click.group(cls=AliasedGroup) +@click.option( + "--environments", + "-e", + help="Overrides the path to the environment file. If not set use the value from your RC file [default: /.environments.json]", + type=click.STRING, + default=".environments.json", +) @verbosity_option() @click.pass_context -def editor(ctx): +def editor(ctx, environments): """beat.editor commands.""" config = ctx.meta["config"] config.set("prefix", os.path.abspath(config.path)) + if not os.path.isabs(environments): + environments = os.path.abspath(os.path.join(config.prefix, environments)) + + ctx.meta["environments"] = environments + global logger logger = setup_logger("beat.editor", ctx.meta["verbosity"]) logger.info("BEAT prefix set to `%s'", ctx.meta["config"].path) + if not os.path.exists(environments): + dump_environments(environments) + QCoreApplication.setApplicationName("beat.editor") QCoreApplication.setOrganizationName("Idiap") QCoreApplication.setOrganizationDomain("ch.idiap") @@ -117,7 +141,7 @@ def edit(ctx, asset_type, asset_name): asset_path = os.path.join( ctx.meta["config"].path, - AssetType.path(AssetType[asset_type.upper()]), + AssetType[asset_type.upper()].path, asset_name + ".json", ) @@ -125,3 +149,22 @@ def edit(ctx, asset_type, asset_name): asset_widget.show() return app.exec_() + + +ENV_REFRESH_EPILOG = """\b +Example: + $ beat editor refresh_env + +""" + + +@editor.command(epilog=ENV_REFRESH_EPILOG) +@click.pass_context +@raise_on_error +def refresh_env(ctx): + """Update environments cache""" + + environments_file_path = ctx.meta["environments"] + if os.path.exists(environments_file_path): + os.remove(environments_file_path) + dump_environments(environments_file_path) -- GitLab From 89a61b520e93de2e36502325c59399bb16ba899d Mon Sep 17 00:00:00 2001 From: Samuel Gaist Date: Fri, 24 May 2019 15:36:18 +0200 Subject: [PATCH 2/3] [scripts][editor_cli] Move environment cache update in option This allows for not calling the update process when --help is passed and allow more fine grained control of which sub command does it. For example, it doesn't make sense to create the cash if the refresh_env sub-command is called. --- beat/editor/scripts/editor_cli.py | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/beat/editor/scripts/editor_cli.py b/beat/editor/scripts/editor_cli.py index 70dc5a1..bf229e5 100644 --- a/beat/editor/scripts/editor_cli.py +++ b/beat/editor/scripts/editor_cli.py @@ -62,6 +62,25 @@ def dump_environments(environments_file_path): logger.info("Done") +def setup_environment_cache(ctx, param, value): + """Click option callback to setup environment cache""" + + if not value: + environments = ctx.meta["environments"] + if not os.path.exists(environments): + dump_environments(environments) + + +refresh_environment_cache_flag = click.option( + "--no-check-env", + is_flag=True, + expose_value=False, + default=False, + help="Do not check for environment cache", + callback=setup_environment_cache, +) + + START_EPILOG = """\b Example: $ beat editor start @@ -95,9 +114,6 @@ def editor(ctx, environments): logger = setup_logger("beat.editor", ctx.meta["verbosity"]) logger.info("BEAT prefix set to `%s'", ctx.meta["config"].path) - if not os.path.exists(environments): - dump_environments(environments) - QCoreApplication.setApplicationName("beat.editor") QCoreApplication.setOrganizationName("Idiap") QCoreApplication.setOrganizationDomain("ch.idiap") @@ -106,6 +122,7 @@ def editor(ctx, environments): @editor.command(epilog=START_EPILOG) @click.pass_context +@refresh_environment_cache_flag @raise_on_error def start(ctx): """Start the beat editor""" @@ -130,6 +147,7 @@ Example: @click.argument("asset_type") @click.argument("asset_name") @click.pass_context +@refresh_environment_cache_flag @raise_on_error def edit(ctx, asset_type, asset_name): """Edit one specific asset""" -- GitLab From 7bb4701ee4c40fdbd18cc73bdc1e7e5c6c87b4ab Mon Sep 17 00:00:00 2001 From: Samuel Gaist Date: Thu, 16 May 2019 15:15:23 +0200 Subject: [PATCH 3/3] [test][conftest] Create a dummy environment cache file Will be used by the ExperimentEditor tests --- beat/editor/test/conftest.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/beat/editor/test/conftest.py b/beat/editor/test/conftest.py index 7a98125..a6ad00e 100644 --- a/beat/editor/test/conftest.py +++ b/beat/editor/test/conftest.py @@ -28,6 +28,7 @@ import sys import tempfile import pytest import pkg_resources +import simplejson as json import subprocess import shutil import importlib @@ -45,6 +46,7 @@ else: prefix = os.path.join(prefix_folder, "prefix") +environment_file = os.path.join(prefix, "environment.json") @pytest.fixture( @@ -64,6 +66,16 @@ def test_prefix(): for path in prefixes: subprocess.check_call(["rsync", "-arz", path, prefix_folder]) + with open(environment_file, "wt") as env_file: + env_file.write( + json.dumps( + { + "Test Env": {"name": "Python 2.7", "version": "1.3.0"}, + "Another test Env": {"name": "another test", "version": "1.1.1"}, + } + ) + ) + yield prefix shutil.rmtree(prefix_folder) @@ -84,7 +96,13 @@ def test_prefix(): @pytest.fixture(scope="module") def beat_context(test_prefix): Context = namedtuple("Context", ["meta"]) - context = Context(meta={"--user": "user", "--prefix": test_prefix}) + context = Context( + meta={ + "--user": "user", + "--prefix": test_prefix, + "environments": environment_file, + } + ) config = Configuration(context.meta) context.meta["config"] = config -- GitLab