Merge branch '61_update_plotterparameter_storage_handling' into 'master'

Update plotterparameter storage handling

Closes #61

See merge request !67

beat/cmdline/plotters.py

@@ -46,6 +46,7 @@ from PIL import Image
 
 from beat.core import plotter
 from beat.core import dataformat
+from beat.core.plotterparameter import Storage as PPStorage
 
 from . import common
 from . import commands
@@ -259,7 +260,7 @@ def plot_impl(
             with open(
                 os.path.join(
                     prefix,
-                    common.TYPE_NOSTORAGE["plotterparameter"],
+                    PPStorage.asset_folder,
                     plotterparameter + ".json",
                 ),
                 "r",

beat/cmdline/test/core.py

@@ -71,6 +71,10 @@ class AssetLocalTest:
         use_cache = kwargs.get("cache", "cache")
         asset_type = kwargs.get("asset_type", cls.asset_type)
 
+        cmd_group = common.TYPE_PLURAL[asset_type]
+        if "/" in cmd_group:
+            cmd_group = cmd_group.split("/")[-1]
+
         runner = CliRunner()
         with runner.isolated_filesystem():
             result = runner.invoke(
@@ -87,7 +91,7 @@ class AssetLocalTest:
                     use_cache,
                     "--platform",
                     use_platform,
-                    common.TYPE_PLURAL[asset_type],
+                    cmd_group,
                 ]
                 + list(args),
                 catch_exceptions=False,

beat/cmdline/test/test_cache.py

@@ -37,67 +37,69 @@
 # Basic tests for the command line beat program: cache
 
 import os
-import click
-from click.testing import CliRunner
 import nose.tools
 
-from . import prefix, tmp_prefix, temp_cwd
-from .utils import index_experiment_dbs
+from click.testing import CliRunner
+
+from beat.core.test.utils import cleanup, slow
 
 from beat.cmdline.scripts import main_cli
 
-from beat.core.test.utils import cleanup, slow
+from .utils import index_experiment_dbs
+
+from . import prefix, tmp_prefix
 
 
 def call(*args, **kwargs):
-  '''A central mechanism to call the main routine with the right parameters'''
+    """A central mechanism to call the main routine with the right parameters"""
 
-  use_prefix = kwargs.get('prefix', prefix)
+    use_prefix = kwargs.get("prefix", prefix)
 
-  runner = CliRunner()
-  with runner.isolated_filesystem():
-    result = runner.invoke(
-        main_cli.main,
-        ['--test-mode', '--prefix', use_prefix, '--cache', tmp_prefix] +
-        list(args)
-    )
-  return result.exit_code, result.output
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        result = runner.invoke(
+            main_cli.main,
+            ["--test-mode", "--prefix", use_prefix, "--cache", tmp_prefix] + list(args),
+            catch_exceptions=False,
+        )
+    return result.exit_code, result.output
 
 
 def setup_module():
-  experiment_name = 'user/user/double_triangle/1/double_triangle'
+    experiment_name = "user/user/double_triangle/1/double_triangle"
+
+    index_experiment_dbs(experiment_name)
 
-  index_experiment_dbs(experiment_name)
+    call("experiments", "run", experiment_name)
 
-  call('experiments', 'run', experiment_name)
 
 def teardown_module():
-  cleanup()
+    cleanup()
 
 
 @slow
 def test_cache_info():
-  assert len(os.listdir(tmp_prefix)) != 0
-  ex_code, out = call('cache', 'info')
-  nose.tools.eq_(ex_code, 0, out)
-  ex_code, out = call('cache', 'info', '--sizes')
-  nose.tools.eq_(ex_code, 0, out)
-  ex_code, out = call('cache', '--start', 0, 'info')
-  nose.tools.eq_(ex_code, 0, out)
+    nose.tools.assert_not_equal(len(os.listdir(tmp_prefix)), 0)
+    ex_code, out = call("cache", "info")
+    nose.tools.eq_(ex_code, 0, out)
+    ex_code, out = call("cache", "info", "--sizes")
+    nose.tools.eq_(ex_code, 0, out)
+    ex_code, out = call("cache", "--start", 0, "info")
+    nose.tools.eq_(ex_code, 0, out)
 
 
 @slow
 def test_cache_view():
-  assert len(os.listdir(tmp_prefix)) != 0
-  ex_code, out = call('cache', 'view')
-  nose.tools.eq_(ex_code, 0, out)
-  ex_code, out = call('cache', '--start', 0, 'view')
-  nose.tools.eq_(ex_code, 0, out)
+    nose.tools.assert_not_equal(len(os.listdir(tmp_prefix)), 0)
+    ex_code, out = call("cache", "view")
+    nose.tools.eq_(ex_code, 0, out)
+    ex_code, out = call("cache", "--start", 0, "view")
+    nose.tools.eq_(ex_code, 0, out)
 
 
 @slow
 def test_cache_clear():
-  assert len(os.listdir(tmp_prefix)) != 0
-  ex_code, out = call('cache', 'clear')
-  nose.tools.eq_(ex_code, 0, out)
-  nose.tools.eq_(len(os.listdir(tmp_prefix)), 0)
+    nose.tools.assert_not_equal(len(os.listdir(tmp_prefix)), 0)
+    ex_code, out = call("cache", "clear")
+    nose.tools.eq_(ex_code, 0, out)
+    nose.tools.eq_(len(os.listdir(tmp_prefix)), 0)

beat/cmdline/test/test_config.py

@@ -38,206 +38,223 @@
 
 import os
 import click
-from click.testing import CliRunner
 import nose.tools
-from nose.tools import assert_raises
 import simplejson
+import shutil
 
-from . import tmp_prefix, temp_cwd
+from click.testing import CliRunner
+
+from beat.core.plotter import Storage as PStorage
 from beat.core.test.utils import cleanup
+
+from beat.cmdline.scripts import main_cli
+
 from .. import config
 from .. import common
-from beat.cmdline.scripts import main_cli
+
+from . import tmp_prefix, temp_cwd
 
 
 def call(*args, **kwargs):
-  '''A central mechanism to call the main routine with the right parameters'''
-  use_prefix = kwargs.get('prefix', tmp_prefix)
+    """A central mechanism to call the main routine with the right parameters"""
+    use_prefix = kwargs.get("prefix", tmp_prefix)
 
-  runner = CliRunner()
-  result = runner.invoke(
-    main_cli.main,
-    ['--test-mode', '--prefix', use_prefix] +
-    list(args)
-  )
-  if result.exit_code != 0:
-      click.echo(result.output)
-  return result.exit_code
+    runner = CliRunner()
+    result = runner.invoke(
+        main_cli.main,
+        ["--test-mode", "--prefix", use_prefix] + list(args),
+        catch_exceptions=False,
+    )
+    if result.exit_code != 0:
+        click.echo(result.output)
+    return result.exit_code
 
 
 def test_config_list():
-  nose.tools.eq_(call('config', 'show'), 0)
+    nose.tools.eq_(call("config", "show"), 0)
 
 
 def test_config_cache():
-  cache_dir = 'cache'
+    cache_dir = "cache"
 
-  c = config.Configuration({'--cache': cache_dir})
-  nose.tools.eq_(c.cache, os.path.join(c.path, cache_dir))
+    c = config.Configuration({"--cache": cache_dir})
+    nose.tools.eq_(c.cache, os.path.join(c.path, cache_dir))
 
-  cache_dir = '/an/absolute/cache/dir'
-  c = config.Configuration({'--cache': cache_dir})
-  nose.tools.eq_(c.cache, cache_dir)
+    cache_dir = "/an/absolute/cache/dir"
+    c = config.Configuration({"--cache": cache_dir})
+    nose.tools.eq_(c.cache, cache_dir)
 
 
 @nose.tools.with_setup(teardown=cleanup)
 def test_set_local_token():
-  token_value = '123456abcdefffff'
-  nose.tools.eq_(call('config', 'set', '--local', 'token', token_value), 0)
-  config = '.beatrc'
-  assert os.path.exists(config)
-  with open(config, 'rt') as f: contents = simplejson.load(f)
-  assert contents['token'] == token_value
+    token_value = "123456abcdefffff"  # nosec
+    nose.tools.eq_(call("config", "set", "--local", "token", token_value), 0)
+    config = ".beatrc"
+    nose.tools.assert_true(os.path.exists(config))
+    with open(config, "rt") as f:
+        contents = simplejson.load(f)
+    nose.tools.eq_(contents["token"], token_value)
 
 
 @nose.tools.with_setup(teardown=cleanup)
 def test_set_local_multiple():
-  token_value = '123456abcde123456abcde123456abcdefff123456abcdef'
-  nose.tools.eq_(call('config', 'set', '--local', 'token', token_value), 0)
-  config = '.beatrc'
-  assert os.path.exists(config)
-  with open(config, 'rt') as f: contents = simplejson.load(f)
-  assert contents['token'] == token_value
-
-  # then we reduce the token size and see if the written file gets messed-up
-  token_value = '123456'
-  nose.tools.eq_(call('config', 'set', '--local', 'token', token_value), 0)
-  assert os.path.exists(config)
-  with open(config, 'rt') as f: contents = simplejson.load(f)
-  assert contents['token'] == token_value
+    token_value = "123456abcde123456abcde123456abcdefff123456abcdef"  # nosec
+    nose.tools.eq_(call("config", "set", "--local", "token", token_value), 0)
+    config = ".beatrc"
+    nose.tools.assert_true(os.path.exists(config))
+    with open(config, "rt") as f:
+        contents = simplejson.load(f)
+    nose.tools.eq_(contents["token"], token_value)
+
+    # then we reduce the token size and see if the written file gets messed-up
+    token_value = "123456"  # nosec
+    nose.tools.eq_(call("config", "set", "--local", "token", token_value), 0)
+    nose.tools.assert_true(os.path.exists(config))
+    with open(config, "rt") as f:
+        contents = simplejson.load(f)
+    nose.tools.eq_(contents["token"], token_value)
 
 
 @nose.tools.with_setup(teardown=cleanup)
 def test_set_local_atnt_db():
-  db_config = 'database/atnt'
-  db_path = './atnt_db'
-  nose.tools.eq_(call('config', 'set', '--local', db_config, db_path), 0)
-  config = '.beatrc'
-  assert os.path.exists(config)
-  with open(config, 'rt') as f: contents = simplejson.load(f)
-  assert contents[db_config] == db_path
+    db_config = "database/atnt"
+    db_path = "./atnt_db"
+    nose.tools.eq_(call("config", "set", "--local", db_config, db_path), 0)
+    config = ".beatrc"
+    nose.tools.assert_true(os.path.exists(config))
+    with open(config, "rt") as f:
+        contents = simplejson.load(f)
+    nose.tools.eq_(contents[db_config], db_path)
 
 
 @nose.tools.with_setup(teardown=cleanup)
 def test_set_get_local_atnt_db():
-  db_config = 'database/atnt'
-  db_path = './atnt_db'
-  nose.tools.eq_(call('config', 'set', '--local', db_config, db_path), 0)
-  nose.tools.eq_(call('config', 'get', db_config), 0)
+    db_config = "database/atnt"
+    db_path = "./atnt_db"
+    nose.tools.eq_(call("config", "set", "--local", db_config, db_path), 0)
+    nose.tools.eq_(call("config", "get", db_config), 0)
 
 
 @nose.tools.with_setup(teardown=cleanup)
 def test_set_bad_config_key():
-  db_config = 'fail'
-  nose.tools.assert_not_equal(call('config', 'set', '--local', db_config, db_config), 0)
+    db_config = "fail"
+    nose.tools.assert_not_equal(
+        call("config", "set", "--local", db_config, db_config), 0
+    )
 
 
 @nose.tools.with_setup(teardown=cleanup)
 def test_get_bad_config_key():
-  db_config = 'fail'
-  nose.tools.assert_not_equal(call('config', 'get', db_config), 0)
+    db_config = "fail"
+    with nose.tools.assert_raises(KeyError):
+        call("config", "get", db_config)
 
 
 @nose.tools.with_setup(teardown=cleanup)
 def test_get_token():
-  nose.tools.eq_(call('config', 'set', '--local', 'token', '12we3f45fgh'), 0)
-  nose.tools.eq_(call('config', 'get', 'token'), 0)
+    nose.tools.eq_(call("config", "set", "--local", "token", "12we3f45fgh"), 0)
+    nose.tools.eq_(call("config", "get", "token"), 0)
 
 
 @nose.tools.with_setup(teardown=cleanup)
 def test_get_editor():
-  nose.tools.eq_(call('config', 'set', '--local', 'editor', 'vi'), 0)
-  nose.tools.eq_(call('config', 'get', 'editor'), 0)
+    nose.tools.eq_(call("config", "set", "--local", "editor", "vi"), 0)
+    nose.tools.eq_(call("config", "get", "editor"), 0)
 
 
 @nose.tools.with_setup(teardown=cleanup)
 def test_set_local_editor():
-  editor_value = 'editor'
-  with temp_cwd() as d:
-    nose.tools.eq_(call('config', 'set', '--local', 'editor', editor_value), 0)
-    config = os.path.join(d, '.beatrc')
-    assert os.path.exists(config)
-    with open(config, 'rt') as f: contents = simplejson.load(f)
-    assert contents['editor'] == editor_value
+    editor_value = "editor"
+    with temp_cwd() as d:
+        nose.tools.eq_(call("config", "set", "--local", "editor", editor_value), 0)
+        config = os.path.join(d, ".beatrc")
+        nose.tools.assert_true(os.path.exists(config))
+        with open(config, "rt") as f:
+            contents = simplejson.load(f)
+        nose.tools.eq_(contents["editor"], editor_value)
 
 
 def create_touch_file(tmp_prefix, editor):
-  cmd = "%s %s && %s %s" % ('mkdir -p', os.path.join(tmp_prefix, 'plotters'), 'touch', os.path.join(tmp_prefix, 'plotters', 'test.py'))
-  os.system(cmd)
-  result = common.edit_local_file(tmp_prefix, editor, 'plotter', "test")
-  return result
+    cmd = "%s %s && %s %s" % (
+        "mkdir -p",
+        os.path.join(tmp_prefix, PStorage.asset_folder),
+        "touch",
+        os.path.join(tmp_prefix, PStorage.asset_folder, "test.py"),
+    )
+    os.system(cmd)  # nosec
+    result = common.edit_local_file(tmp_prefix, editor, "plotter", "test")
+    return result
 
 
 def read_data(tmp_prefix):
-  with open(os.path.join(tmp_prefix, 'plotters', 'test.py'), 'r') as f:
-      read_data = f.read().split('\n')[0]
-  f.closed
-  return read_data
+    with open(os.path.join(tmp_prefix, PStorage.asset_folder, "test.py"), "r") as f:
+        read_data = f.read().split("\n")[0]
+    f.closed
+    return read_data
 
 
 def clean_tmp_files(tmp_prefix):
-  cmd = "%s %s" % ('rm -fr', os.path.join(tmp_prefix, 'plotters'))
-  os.system(cmd)
+    shutil.rmtree(os.path.join(tmp_prefix, PStorage.asset_folder))
 
 
 @nose.tools.with_setup(teardown=cleanup)
 def test_check_editor_system_no_editor_set():
-  editor = None
-  os.environ['VISUAL'] = ''
-  os.environ['EDITOR'] = ''
+    editor = None
+    os.environ["VISUAL"] = ""
+    os.environ["EDITOR"] = ""
 
-  result = create_touch_file(tmp_prefix, editor)
-  assert result == 1
+    result = create_touch_file(tmp_prefix, editor)
+    nose.tools.eq_(result, 1)
 
-  data = read_data(tmp_prefix)
-  assert len(data) == 0
+    data = read_data(tmp_prefix)
+    nose.tools.eq_(len(data), 0)
 
-  clean_tmp_files(tmp_prefix)
+    clean_tmp_files(tmp_prefix)
 
 
 @nose.tools.with_setup(teardown=cleanup)
 def test_check_editor_system_no_local_editor():
-  editor = None
-  os.environ['VISUAL'] = 'echo "2" >'
-  os.environ['EDITOR'] = 'echo "3" >'
+    editor = None
+    os.environ["VISUAL"] = 'echo "2" >'
+    os.environ["EDITOR"] = 'echo "3" >'
 
-  result = create_touch_file(tmp_prefix, editor)
-  assert result == 0
+    result = create_touch_file(tmp_prefix, editor)
+    nose.tools.eq_(result, 0)
 
-  data = read_data(tmp_prefix)
-  assert len(data) == 1
-  assert data == "2"
+    data = read_data(tmp_prefix)
+    nose.tools.eq_(len(data), 1)
+    nose.tools.eq_(data, "2")
 
-  clean_tmp_files(tmp_prefix)
+    clean_tmp_files(tmp_prefix)
 
 
 @nose.tools.with_setup(teardown=cleanup)
 def test_check_editor_system_local_editor_set():
-  editor = 'echo "1" >'
-  os.environ['VISUAL'] = 'echo "2" >'
-  os.environ['EDITOR'] = 'echo "3" >'
+    editor = 'echo "1" >'
+    os.environ["VISUAL"] = 'echo "2" >'
+    os.environ["EDITOR"] = 'echo "3" >'
 
-  result = create_touch_file(tmp_prefix, editor)
-  assert result == 0
+    result = create_touch_file(tmp_prefix, editor)
+    nose.tools.eq_(result, 0)
 
-  data = read_data(tmp_prefix)
-  assert len(data) == 1
-  assert data == "1"
+    data = read_data(tmp_prefix)
+    nose.tools.eq_(len(data), 1)
+    nose.tools.eq_(data, "1")
 
-  clean_tmp_files(tmp_prefix)
+    clean_tmp_files(tmp_prefix)
 
 
 @nose.tools.with_setup(teardown=cleanup)
 def test_check_editor_system_no_local_editor_no_visual():
-  editor = None
-  os.environ['VISUAL'] = ''
-  os.environ['EDITOR'] = 'echo "3" >'
+    editor = None
+    os.environ["VISUAL"] = ""
+    os.environ["EDITOR"] = 'echo "3" >'
 
-  result = create_touch_file(tmp_prefix, editor)
-  assert result == 0
+    result = create_touch_file(tmp_prefix, editor)
+    nose.tools.eq_(result, 0)
 
-  data = read_data(tmp_prefix)
-  assert len(data) == 1
-  assert data == "3"
+    data = read_data(tmp_prefix)
+    nose.tools.eq_(len(data), 1)
+    nose.tools.eq_(data, "3")
 
-  clean_tmp_files(tmp_prefix)
+    clean_tmp_files(tmp_prefix)

beat/cmdline/test/test_databases.py

@@ -75,6 +75,7 @@ def call(*args, **kwargs):
                 "databases",
             ]
             + list(args),
+            catch_exceptions=False,
         )
     if result.exit_code != 0:
         click.echo(result.output)

beat/cmdline/test/test_dataformats.py

@@ -71,6 +71,7 @@ def call(*args, **kwargs):
                 "dataformats",
             ]
             + list(args),
+            catch_exceptions=False,
         )
     if result.exit_code != 0:
         click.echo(result.output)

beat/cmdline/test/test_experiments.py

@@ -80,6 +80,7 @@ def call(*args, **kwargs):
                 "experiments",
             ]
             + list(args),
+            catch_exceptions=False,
         )
     if result.exit_code != 0:
         click.echo(result.output)

beat/cmdline/test/test_plotterparameters.py

@@ -37,50 +37,78 @@
 # Basic tests for the command line beat program: plotterparameters
 
 import os
-import sys
 import nose.tools
-import shutil
-import json
 import click
+
 from click.testing import CliRunner
 
-from . import platform, disconnected, prefix, tmp_prefix, user, token, temp_cwd
-from .utils import index_experiment_dbs
-from ..common import Selector
+from beat.core.plotterparameter import Storage
 from beat.core.test.utils import slow, cleanup, skipif
+
 from beat.cmdline.scripts import main_cli
 
+from . import core
+from . import platform, disconnected, prefix, tmp_prefix, user, token
+
 
 def call(*args, **kwargs):
-  '''A central mechanism to call the main routine with the right parameters'''
-
-  use_prefix = kwargs.get('prefix', prefix)
-  use_platform = kwargs.get('platform', platform)
-  use_cache = kwargs.get('cache', 'cache')
-
-  runner = CliRunner()
-  with runner.isolated_filesystem():
-    result = runner.invoke(
-        main_cli.main,
-        ['--test-mode', '--platform', use_platform,
-        '--prefix', use_prefix, '--token', token, '--user', user,
-        '--cache', use_cache, 'plotterparameters'] +
-        list(args)
-    )
-  if result.exit_code != 0:
-      click.echo(result.output)
-  return result.exit_code
+    """A central mechanism to call the main routine with the right parameters"""
+
+    use_prefix = kwargs.get("prefix", prefix)
+    use_platform = kwargs.get("platform", platform)
+    use_cache = kwargs.get("cache", "cache")
+
+    runner = CliRunner()
+    with runner.isolated_filesystem():
+        result = runner.invoke(
+            main_cli.main,
+            [
+                "--test-mode",
+                "--platform",
+                use_platform,
+                "--prefix",
+                use_prefix,
+                "--token",
+                token,
+                "--user",
+                user,
+                "--cache",
+                use_cache,
+                "plotterparameters",
+            ]
+            + list(args),
+            catch_exceptions=False,
+        )
+    if result.exit_code != 0:
+        click.echo(result.output)