Skip to content
GitLab
Commit ea940304 authored by Samuel GAIST's avatar Samuel GAIST
Browse files

[common] Remove all code related to no storage asset and fix usage type keyword

parent a830c44a
Show whitespace changes
Inline Side-by-side
beat/cmdline/common.py
View file @ ea940304
......@@ -59,8 +59,6 @@ from beat.core import algorithm
from beat.core import toolchain
from beat.core import experiment
from beat.backend.python import utils
logger = logging.getLogger(__name__)
TYPE_GLOB = {
......@@ -113,10 +111,6 @@ TYPE_STORAGE = {
"experiment": experiment.Storage,
}
NOSTORAGE = []
TYPE_NOSTORAGE = {}
TYPE_PLURAL = {
"dataformat": "dataformats",
"database": "databases",
......@@ -185,13 +179,13 @@ class Selector(object):
def __ensure_entries(self):
"""Ensure all types have an entry"""
for type_ in self.__versionables:
if type_ not in self.__version:
self.__version[type_] = dict()
for asset_type in self.__versionables:
if asset_type not in self.__version:
self.__version[asset_type] = dict()
for type_ in self.__forkables:
if type_ not in self.__fork:
self.__fork[type_] = dict()
for asset_type in self.__forkables:
if asset_type not in self.__fork:
self.__fork[asset_type] = dict()
def fork(self, type, src, dst):
"""Registers that object ``dst`` is a fork of object ``src``"""
......@@ -210,36 +204,36 @@ class Selector(object):
return self.__fork[type].get(name)
def version(self, type_, src, dst):
def version(self, asset_type, src, dst):
"""Registers that object ``dst`` is a new version of object ``src``"""
if not type_ in self.__version:
raise RuntimeError("Can't create new version of {}".format(type_))
if asset_type not in self.__version:
raise RuntimeError("Can't create new version of {}".format(asset_type))
logger.info(
"`%s/%s' is a new version of `%s/%s'",
TYPE_PLURAL[type_],
TYPE_PLURAL[asset_type],
dst,
TYPE_PLURAL[type_],
TYPE_PLURAL[asset_type],
src,
)
self.__version[type_][dst] = src
self.__version[asset_type][dst] = src
def version_of(self, type_, name):
def version_of(self, asset_type, name):
"""Returns the name of the originating version object or ``None``"""
if type_ not in self.__version:
if asset_type not in self.__version:
return None
return self.__version[type_].get(name)
return self.__version[asset_type].get(name)
def delete(self, type_, name):
def delete(self, asset_type, name):
"""Forgets about an object that was being tracked"""
if name in self.__fork[type_]:
del self.__fork[type_][name]
if type_ in self.__version and name in self.__version[type_]:
del self.__version[type_][name]
if name in self.__fork[asset_type]:
del self.__fork[asset_type][name]
if asset_type in self.__version and name in self.__version[asset_type]:
del self.__version[asset_type][name]
def load(self):
"""Loads contents from file"""
......@@ -273,7 +267,7 @@ class Selector(object):
simplejson.dump(data, f, indent=2)
def retrieve_remote_list(webapi, type, fields):
def retrieve_remote_list(webapi, asset_type, fields):
"""Utility function used by commands to retrieve a remote list of objects
......@@ -282,7 +276,7 @@ def retrieve_remote_list(webapi, type, fields):
webapi (object): An instance of our WebAPI class, prepared to access the
BEAT server of interest
type (str): One of ``database``, ``dataformat``, ``algorithm``,
asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
``toolchain`` or ``experiment``.
fields (:py:class:`list`): A list of fields to retrieve from the remote
......@@ -296,11 +290,11 @@ def retrieve_remote_list(webapi, type, fields):
"""
logger.debug("retrieving remote %s list...", TYPE_PLURAL[type])
logger.debug("retrieving remote %s list...", TYPE_PLURAL[asset_type])
fields = "" if not fields else "?fields=%s" % ",".join(fields)
url = "/api/v1/%s/%s" % (TYPE_PLURAL[type], fields)
url = "/api/v1/%s/%s" % (TYPE_PLURAL[asset_type], fields)
(status_code, content) = webapi.get(url)
if status_code is None:
......@@ -309,7 +303,7 @@ def retrieve_remote_list(webapi, type, fields):
if status_code != six.moves.http_client.OK:
logger.error(
"failed to retrieve %s list from `%s' on behalf of `%s', " "reason: %s",
TYPE_PLURAL[type],
TYPE_PLURAL[asset_type],
webapi.platform,
webapi.user,
six.moves.http_client.responses[status_code],
......@@ -319,7 +313,7 @@ def retrieve_remote_list(webapi, type, fields):
return simplejson.loads(content, object_pairs_hook=collections.OrderedDict)
def make_up_remote_list(webapi, type, requirements):
def make_up_remote_list(webapi, asset_type, requirements):
"""Creates a list of downloadable objects from user requirements.
This function can create a list of downloadable objects from user
......@@ -334,7 +328,7 @@ def make_up_remote_list(webapi, type, requirements):
webapi (object): An instance of our WebAPI class, prepared to access the
BEAT server of interest
type (str): One of ``database``, ``dataformat``, ``algorithm``,
asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
``toolchain`` or ``experiment``.
requirements (:py:class:`list`): A list of requirements that are used to
......@@ -348,7 +342,7 @@ def make_up_remote_list(webapi, type, requirements):
"""
candidates = retrieve_remote_list(webapi, type, ["name"])
candidates = retrieve_remote_list(webapi, asset_type, ["name"])
if not requirements: # special case, return all possible values
if candidates is None:
......@@ -356,7 +350,7 @@ def make_up_remote_list(webapi, type, requirements):
return [c["name"] for c in candidates]
# othewise, we need to separate filters from full-names
full_requirements = fnmatch.filter(requirements, TYPE_FNMATCH[type])
full_requirements = fnmatch.filter(requirements, TYPE_FNMATCH[asset_type])
short_requirements = [k for k in requirements if k not in full_requirements]
retval = []
......@@ -377,7 +371,7 @@ def make_up_remote_list(webapi, type, requirements):
return retval + full_requirements
def display_remote_list(webapi, type):
def display_remote_list(webapi, asset_type):
"""Implements a generic "list --remote" command
Parameters:
......@@ -385,7 +379,7 @@ def display_remote_list(webapi, type):
webapi (object): An instance of our WebAPI class, prepared to access the
BEAT server of interest, on behalf of a pre-configured user.
type (str): One of ``database``, ``dataformat``, ``algorithm``,
asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
``toolchain`` or ``experiment``.
......@@ -397,7 +391,9 @@ def display_remote_list(webapi, type):
"""
remote_list = retrieve_remote_list(webapi, type, ["name", "short_description"])
remote_list = retrieve_remote_list(
webapi, asset_type, ["name", "short_description"]
)
if remote_list is None:
return 1
......@@ -407,14 +403,14 @@ def display_remote_list(webapi, type):
logger.extra(2 * " " + item["short_description"])
if len(remote_list) != 1:
logger.extra("%d %s found", len(remote_list), TYPE_PLURAL[type])
logger.extra("%d %s found", len(remote_list), TYPE_PLURAL[asset_type])
else:
logger.extra("1 %s found" % type)
logger.extra("1 %s found" % asset_type)
return 0
def make_up_local_list(prefix, type, requirements):
def make_up_local_list(prefix, asset_type, requirements):
"""Creates a list of uploadable objects from user requirements.
This function can create a list of uploadable objects from user requirements.
......@@ -429,7 +425,7 @@ def make_up_local_list(prefix, type, requirements):
prefix (str): A string representing the root of the path in which the user
objects are stored
type (str): One of ``database``, ``dataformat``, ``algorithm``,
asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
``toolchain`` or ``experiment``.
requirements (:py:class:`list`): A list of requirements that are used to
......@@ -444,21 +440,14 @@ def make_up_local_list(prefix, type, requirements):
"""
root = os.path.join(prefix, TYPE_PLURAL[type])
if type in NOSTORAGE:
try:
root = os.path.join(prefix, TYPE_NOSTORAGE[type])
except IndexError:
logger.error("Selected type is not valid: %s", type)
return 1
asset_path_list = glob.glob(os.path.join(root, TYPE_GLOB[type]))
root = os.path.join(prefix, TYPE_PLURAL[asset_type])
asset_path_list = glob.glob(os.path.join(root, TYPE_GLOB[asset_type]))
candidates = [
os.path.splitext(os.path.relpath(path, root))[0] for path in asset_path_list
]
# adds hashed path structures
hashed_path_list = glob.glob(os.path.join(root, "*", "*", TYPE_GLOB[type]))
hashed_path_list = glob.glob(os.path.join(root, "*", "*", TYPE_GLOB[asset_type]))
hashed_path_list = [
os.path.splitext(os.path.relpath(path, root))[0] for path in hashed_path_list
]
......@@ -469,13 +458,13 @@ def make_up_local_list(prefix, type, requirements):
use_requirements = []
for k in requirements: # remove leading plural-name
if k.startswith(TYPE_PLURAL[type] + os.sep):
use_requirements.append(k.replace(TYPE_PLURAL[type] + os.sep, ""))
if k.startswith(TYPE_PLURAL[asset_type] + os.sep):
use_requirements.append(k.replace(TYPE_PLURAL[asset_type] + os.sep, ""))
else:
use_requirements.append(k)
requirements = use_requirements
full_requirements = fnmatch.filter(requirements, TYPE_FNMATCH[type])
full_requirements = fnmatch.filter(requirements, TYPE_FNMATCH[asset_type])
short_requirements = [k for k in requirements if k not in full_requirements]
retval = oset.oset()
......@@ -487,7 +476,7 @@ def make_up_local_list(prefix, type, requirements):
return list(retval) + full_requirements
def display_local_list(prefix, type):
def display_local_list(prefix, asset_type):
"""Implements the local "list" command
......@@ -496,7 +485,7 @@ def display_local_list(prefix, type):
prefix (str): A string representing the root of the path in which the user
objects are stored
type (str): One of ``database``, ``dataformat``, ``algorithm``,
asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
``toolchain`` or ``experiment``.
......@@ -508,21 +497,12 @@ def display_local_list(prefix, type):
"""
names = make_up_local_list(prefix, type, [])
names = make_up_local_list(prefix, asset_type, [])
for name in names:
logger.info("%s", name)
try:
contents = None
if type in NOSTORAGE:
name_path = os.path.join(prefix, TYPE_NOSTORAGE[type], name)
data_json = utils.File(name_path + ".json")
with open(data_json.path, "rt") as f:
contents = simplejson.load(
f, object_pairs_hook=collections.OrderedDict
)
else:
storage = TYPE_STORAGE[type](prefix, name)
storage = TYPE_STORAGE[asset_type](prefix, name)
contents = simplejson.loads(
storage.json.load(), object_pairs_hook=collections.OrderedDict
)
......@@ -532,14 +512,14 @@ def display_local_list(prefix, type):
logger.warn(2 * " " + "(!) invalid JSON file")
if len(names) != 1:
logger.extra("%d %s found", len(names), TYPE_PLURAL[type])
logger.extra("%d %s found", len(names), TYPE_PLURAL[asset_type])
else:
logger.extra("1 %s found" % type)
logger.extra("1 %s found" % asset_type)
return 0
def display_local_path(prefix, type, names):
def display_local_path(prefix, asset_type, names):
"""Implements the local "path" command
......@@ -548,7 +528,7 @@ def display_local_path(prefix, type, names):
prefix (str): A string representing the root of the path in which the user
objects are stored
type (str): One of ``database``, ``dataformat``, ``algorithm``,
asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
``toolchain`` or ``experiment``.
......@@ -562,14 +542,11 @@ def display_local_path(prefix, type, names):
selected_type = None
if type not in NOSTORAGE:
try:
selected_type = TYPE_PLURAL[type]
selected_type = TYPE_PLURAL[asset_type]
except IndexError:
logger.error("Selected type is not valid: %s", type)
logger.error("Selected type is not valid: %s", asset_type)
return 1
else:
selected_type = TYPE_NOSTORAGE[type]
for name in names:
root = os.path.join(prefix, selected_type)
......@@ -598,7 +575,7 @@ def display_local_path(prefix, type, names):
return 0
def edit_local_file(prefix, editor, type, name):
def edit_local_file(prefix, editor, asset_type, name):
"""Implements the local "path" command
......@@ -607,7 +584,7 @@ def edit_local_file(prefix, editor, type, name):
prefix (str): A string representing the root of the path in which the user
objects are stored
type (str): One of ``database``, ``dataformat``, ``algorithm``,
asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
``toolchain`` or ``experiment``.
......@@ -621,14 +598,11 @@ def edit_local_file(prefix, editor, type, name):
selected_type = None
if type not in NOSTORAGE:
try:
selected_type = TYPE_PLURAL[type]
selected_type = TYPE_PLURAL[asset_type]
except IndexError:
logger.error("Selected type is not valid: %s", type)
logger.error("Selected type is not valid: %s", asset_type)
return 1
else:
selected_type = TYPE_NOSTORAGE[type]
python_objects = ["database", "library", "algorithm", "plotter"]
json_objects = [
......@@ -640,12 +614,12 @@ def edit_local_file(prefix, editor, type, name):
]
ext = None
if type in python_objects:
if asset_type in python_objects:
ext = ".py"
elif type in json_objects:
elif asset_type in json_objects:
ext = ".json"
else:
logger.error("Selected type is not valid: %s", type)
logger.error("Selected type is not valid: %s", asset_type)
root = os.path.join(prefix, selected_type)
object_path = os.path.join(root, name + ext)
......@@ -714,10 +688,10 @@ def make_webapi(c):
def __exit__(self, *exc):
pass
def _message(self, type, url, data=None):
def _message(self, asset_type, url, data=None):
if data:
data = simplejson.dumps(data)
retval = getattr(super(APIClientContext, self), type)(
retval = getattr(super(APIClientContext, self), asset_type)(
url, data, content_type="application/json"
)
return (retval.status_code, retval.content)
......@@ -743,7 +717,7 @@ def make_webapi(c):
return WebAPI(c.platform, c.user, c.token)
def check_one(prefix, type, name):
def check_one(prefix, asset_type, name):
"""Implements object validation for a single, well-defined object
......@@ -752,7 +726,7 @@ def check_one(prefix, type, name):
prefix (str): A string representing the root of the path in which the user
objects are stored
type (str): One of ``database``, ``dataformat``, ``algorithm``,
asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
``toolchain`` or ``experiment``.
name (str): The name of the object, representing the unique relative path
......@@ -763,20 +737,20 @@ def check_one(prefix, type, name):
"""
o = TYPE_VALIDATOR[type](prefix, name)
o = TYPE_VALIDATOR[asset_type](prefix, name)
if not o.valid:
logger.info("%s/%s [invalid]", TYPE_PLURAL[type], name)
logger.info("%s/%s [invalid]", TYPE_PLURAL[asset_type], name)
for e in o.errors:
logger.warn(" * %s", e)
return 1
else:
logger.info("%s/%s [ok]", TYPE_PLURAL[type], name)
logger.info("%s/%s [ok]", TYPE_PLURAL[asset_type], name)
return 0
def check(prefix, type, names):
def check(prefix, asset_type, names):
"""Implements object validation
......@@ -785,7 +759,7 @@ def check(prefix, type, names):
prefix (str): A string representing the root of the path in which the user
objects are stored
type (str): One of ``database``, ``dataformat``, ``algorithm``,
asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
``toolchain`` or ``experiment``.
names (:py:class:`list`): A list of strings, each representing the unique
......@@ -801,11 +775,11 @@ def check(prefix, type, names):
"""
names = make_up_local_list(prefix, type, names)
return sum([check_one(prefix, type, name) for name in names])
names = make_up_local_list(prefix, asset_type, names)
return sum([check_one(prefix, asset_type, name) for name in names])
def fetch_object(webapi, type, name, fields):
def fetch_object(webapi, asset_type, name, fields):
"""Retrieves a single well-known object from the server
Parameters:
......@@ -813,7 +787,7 @@ def fetch_object(webapi, type, name, fields):
webapi (object): An instance of our WebAPI class, prepared to access the
BEAT server of interest
type (str): One of ``database``, ``dataformat``, ``algorithm``,
asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
``toolchain`` or ``experiment``.
name (str): A string defining the name of the object to retrieve
......@@ -830,9 +804,9 @@ def fetch_object(webapi, type, name, fields):
fields = "?object_format=string&fields=%s" % ",".join(fields)
if name is not None:
url = "/api/v1/%s/%s/%s" % (TYPE_PLURAL[type], name, fields)
url = "/api/v1/%s/%s/%s" % (TYPE_PLURAL[asset_type], name, fields)
else:
url = "/api/v1/%s/%s" % (TYPE_PLURAL[type], fields)
url = "/api/v1/%s/%s" % (TYPE_PLURAL[asset_type], fields)
(status_code, content) = webapi.get(url)
if status_code is None:
......@@ -841,7 +815,7 @@ def fetch_object(webapi, type, name, fields):
if status_code != six.moves.http_client.OK:
logger.error(
"failed to retrieve %s from `%s' with secret token, " "reason: %s",
type,
asset_type,
webapi.platform,
six.moves.http_client.responses[status_code],
)
......@@ -850,7 +824,7 @@ def fetch_object(webapi, type, name, fields):
return simplejson.loads(content, object_pairs_hook=collections.OrderedDict)
def pull(webapi, prefix, type, names, fields, force, indentation):
def pull(webapi, prefix, asset_type, names, fields, force, indentation):
"""Copies objects from the server to the local prefix
......@@ -862,7 +836,7 @@ def pull(webapi, prefix, type, names, fields, force, indentation):
prefix (str): A string representing the root of the path in which the user
objects are stored
type (str): One of ``database``, ``dataformat``, ``algorithm``,
asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
``toolchain`` or ``experiment``.
names (:py:class:`list`): A list of strings, each representing the unique
......@@ -898,7 +872,7 @@ def pull(webapi, prefix, type, names, fields, force, indentation):
"""
names = make_up_remote_list(webapi, type, names)
names = make_up_remote_list(webapi, asset_type, names)
if not names:
return 1, []
indent = indentation * " "
......@@ -907,83 +881,40 @@ def pull(webapi, prefix, type, names, fields, force, indentation):
status = 0
for name in names:
if type not in NOSTORAGE:
# typical workflow with normal storage
storage = TYPE_STORAGE[type](prefix, name)
storage = TYPE_STORAGE[asset_type](prefix, name)
if storage.exists() and not force: # exists locally, force not set
logger.extra(
"%sskipping download of `%s/%s' (exists locally)",
indent,
TYPE_PLURAL[type],
TYPE_PLURAL[asset_type],
name,
)
available.add(name)
continue
else:
logger.info("%sretrieving `%s/%s'...", indent, TYPE_PLURAL[type], name)
data = fetch_object(webapi, type, name, fields)
logger.info(
"%sretrieving `%s/%s'...", indent, TYPE_PLURAL[asset_type], name
)
data = fetch_object(webapi, asset_type, name, fields)
if data is None:
status += 1 # error
continue
if type == "plotterparameter":
if asset_type == "plotterparameter":
declaration = {
"description": data["short_description"],
"plotter": data["plotter"],
"data": data["data"]
"data": data["data"],
}
storage.save(declaration)
else:
storage.save(**data)
available.add(name)
else:
# other workflow with no storage (i.e.: plotterparameter)
# check if storage exists
path = os.path.join(prefix, TYPE_NOSTORAGE[type], name.rsplit("/", 1)[0])
if os.path.exists(path) and not force:
logger.extra(
"%sskipping download of `%s/%s' (exists locally)",
indent,
TYPE_PLURAL[type],
name,
)
available.add(name)
continue
else:
# create folder
if not os.path.exists(path):
os.makedirs(path)
logger.info("%sretrieving `%s/%s'...", indent, TYPE_PLURAL[type], name)
data = fetch_object(webapi, type, name, fields)
if data is None:
status += 1 # error
continue
if "data" in fields:
name_path = os.path.join(prefix, TYPE_NOSTORAGE[type], name)
data_json = utils.File(name_path + ".json")
_buf_data = {}
if type == "plotterparameter":
# generate plotterparameter data
_buf_data["plotter"] = data["plotter"]
_buf_data["data"] = data["data"]
_buf_data["description"] = data["short_description"]
_data = simplejson.dumps(_buf_data, indent=4)
else:
_buf_data["data"] = data["data"]
_data = simplejson.dumps(_buf_data["data"], indent=4)
data_json.save(_data)
if "short_description" in fields:
name_path = os.path.join(prefix, TYPE_NOSTORAGE[type], name)
data_doc = utils.File(name_path + ".rst")
data_doc.save(data["short_description"].encode("utf8"))
available.add(name)
return status, list(available)
def diff(webapi, prefix, type_, name, fields):
def diff(webapi, prefix, asset_type, name, fields):
"""Shows the differences between two objects, for each of the fields
......@@ -995,7 +926,7 @@ def diff(webapi, prefix, type_, name, fields):
prefix (str): A string representing the root of the path in which the user
objects are stored
type (str): One of ``database``, ``dataformat``, ``algorithm``,
asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
``toolchain`` or ``experiment``.
name (str): A string defining the name of the object to calculate
......@@ -1028,8 +959,8 @@ def diff(webapi, prefix, type_, name, fields):
return difflib.unified_diff(
remote.split("\n"),
local.split("\n"),
os.path.join("remote", type_, name + ext),
os.path.join("local", type_, name + ext),
os.path.join("remote", asset_type, name + ext),
os.path.join("local", asset_type, name + ext),