Commit 3a1fdc1d authored by André Anjos's avatar André Anjos 💬

Merge branch 'fix_protocol_template_handling' into 'master'

Fix protocol template handling

See merge request !107
parents 20596949 0e7bc093
Pipeline #47525 passed with stages
in 6 minutes and 37 seconds
...@@ -48,6 +48,7 @@ from beat.core import inputs ...@@ -48,6 +48,7 @@ from beat.core import inputs
from beat.core import utils from beat.core import utils
from beat.core.data import RemoteDataSource from beat.core.data import RemoteDataSource
from beat.core.database import Database from beat.core.database import Database
from beat.core.database import Storage
from beat.core.hash import hashDataset from beat.core.hash import hashDataset
from beat.core.hash import toPath from beat.core.hash import toPath
from beat.core.utils import NumpyJSONEncoder from beat.core.utils import NumpyJSONEncoder
...@@ -363,6 +364,7 @@ def pull_impl(webapi, prefix, names, force, indentation, format_cache): ...@@ -363,6 +364,7 @@ def pull_impl(webapi, prefix, names, force, indentation, format_cache):
""" """
from .dataformats import pull_impl as dataformats_pull from .dataformats import pull_impl as dataformats_pull
from .protocoltemplates import pull_impl as protocoltemplates_pull
status, names = common.pull( status, names = common.pull(
webapi, webapi,
...@@ -374,6 +376,22 @@ def pull_impl(webapi, prefix, names, force, indentation, format_cache): ...@@ -374,6 +376,22 @@ def pull_impl(webapi, prefix, names, force, indentation, format_cache):
indentation, indentation,
) )
# A database object cannot properly loaded if its protocol templates are
# missing, therefore, we must use lower level access.
protocol_templates = set()
for name in names:
db = Storage(prefix, name)
declaration, _, _ = db.load()
declaration = simplejson.loads(declaration)
version = declaration.get("schema_version", 1)
if version > 1:
for protocol in declaration["protocols"]:
protocol_templates.add(protocol["template"])
pt_status = protocoltemplates_pull(
webapi, prefix, protocol_templates, force, indentation + 2, format_cache
)
# see what dataformats one needs to pull # see what dataformats one needs to pull
dataformats = [] dataformats = []
for name in names: for name in names:
...@@ -385,7 +403,7 @@ def pull_impl(webapi, prefix, names, force, indentation, format_cache): ...@@ -385,7 +403,7 @@ def pull_impl(webapi, prefix, names, force, indentation, format_cache):
webapi, prefix, dataformats, force, indentation + 2, format_cache webapi, prefix, dataformats, force, indentation + 2, format_cache
) )
return status + df_status return status + df_status + pt_status
# ---------------------------------------------------------- # ----------------------------------------------------------
......
...@@ -38,7 +38,10 @@ import logging ...@@ -38,7 +38,10 @@ import logging
import click import click
from beat.core import protocoltemplate
from . import commands from . import commands
from . import common
from .click_helper import AliasedGroup from .click_helper import AliasedGroup
from .click_helper import AssetCommand from .click_helper import AssetCommand
from .click_helper import AssetInfo from .click_helper import AssetInfo
...@@ -62,3 +65,74 @@ def protocoltemplates(ctx): ...@@ -62,3 +65,74 @@ def protocoltemplates(ctx):
CMD_LIST = ["list", "path", "edit", "check", "create", "version", ("rm", "rm_local")] CMD_LIST = ["list", "path", "edit", "check", "create", "version", ("rm", "rm_local")]
commands.initialise_asset_commands(protocoltemplates, CMD_LIST, ProtocolTemplateCommand) commands.initialise_asset_commands(protocoltemplates, CMD_LIST, ProtocolTemplateCommand)
def pull_impl(webapi, prefix, names, force, indentation, cache):
"""Copies protocol template from the server
Parameters:
webapi (object): An instance of our WebAPI class, prepared to access the
BEAT server of interest
prefix (str): A string representing the root of the path in which the user
objects are stored
names (:py:class:`list`): A list of strings, each representing the unique
relative path of the objects to retrieve or a list of usernames from
which to retrieve objects. If the list is empty, then we pull all
available objects of a given type. If no user is set, then pull all
public objects of a given type.
force (bool): If set to ``True``, then overwrites local changes with the
remotely retrieved copies.
indentation (int): The indentation level, useful if this function is called
recursively while downloading different object types. This is normally
set to ``0`` (zero).
Returns:
int: Indicating the exit status of the command, to be reported back to the
calling process. This value should be zero if everything works OK,
otherwise, different than zero (POSIX compliance).
"""
from .dataformats import pull_impl as dataformats_pull
protocoltemplates = set(names)
if not protocoltemplates:
return 0
status, downloaded = common.pull(
webapi,
prefix,
"protocoltemplate",
protocoltemplates,
["declaration", "description"],
force,
indentation,
)
if status != 0:
return status
if indentation == 0:
indentation = 4
# see what else one needs to pull
dataformats = set()
for name in protocoltemplates:
try:
obj = protocoltemplate.ProtocolTemplate(prefix, name)
dataformats |= obj.dataformats.keys()
except Exception as e:
logger.error("loading `%s': %s...", name, str(e))
cache[name] = None
df_status = dataformats_pull(
webapi, prefix, dataformats, force, indentation + 2, cache
)
return status + df_status
...@@ -227,3 +227,24 @@ class TestDockerExperimentRun(core.AssetBaseTest): ...@@ -227,3 +227,24 @@ class TestDockerExperimentRun(core.AssetBaseTest):
nose.tools.eq_(exit_code, 1, outputs) nose.tools.eq_(exit_code, 1, outputs)
exit_code, outputs = self.call("run", obj, "--docker", cache=tmp_prefix) exit_code, outputs = self.call("run", obj, "--docker", cache=tmp_prefix)
nose.tools.eq_(exit_code, 1, outputs) nose.tools.eq_(exit_code, 1, outputs)
@slow
def test_run_raw_data_access(self):
"""Test that the raw data access works"""
db = Database(prefix, "simple_rawdata_access/1")
nose.tools.assert_true(db.valid, db.errors)
data_sharing_path = db.data["root_folder"]
offset = 12
with open(os.path.join(data_sharing_path, "datafile.txt"), "wt") as data_file:
data_file.write("{}".format(offset))
obj = "user/user/single/1/single_rawdata_access"
args = ["run", obj, "--docker"]
exit_code, outputs = self.call(*args, cache=tmp_prefix)
nose.tools.eq_(exit_code, 0, outputs)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment