Commit 74f87c14 authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

don't break existing code

parent def2b9f8
Pipeline #41086 passed with stage
in 3 minutes and 21 seconds
......@@ -225,6 +225,40 @@ class Analyzer_(Algorithm_):
# ----------------------------------------------------------
class Storage(utils.CodeStorage):
"""Resolves paths for algorithms
Parameters:
prefix (str): Establishes the prefix of your installation.
name (str): The name of the algorithm object in the format
``<user>/<name>/<version>``.
"""
asset_type = "algorithm"
asset_folder = "algorithms"
def __init__(self, prefix, name, language=None):
if name.count("/") != 2:
raise RuntimeError("invalid algorithm name: `%s'" % name)
self.username, self.name, self.version = name.split("/")
self.fullname = name
self.prefix = prefix
path = utils.hashed_or_simple(
self.prefix, self.asset_folder, name, suffix=".json"
)
path = path[:-5]
super(Storage, self).__init__(path, language)
# ----------------------------------------------------------
class Runner(object):
"""A special loader class for algorithms, with specialized methods
......@@ -344,8 +378,8 @@ class Runner(object):
# The method is optional
if hasattr(self.obj, "prepare"):
if self.algorithm.type in [
Algorithm_.AUTONOMOUS,
Algorithm_.AUTONOMOUS_LOOP_PROCESSOR,
Algorithm.AUTONOMOUS,
Algorithm.AUTONOMOUS_LOOP_PROCESSOR,
]:
self.prepared = loader.run(
self.obj, "prepare", self.exc, data_loaders.secondaries()
......@@ -382,14 +416,14 @@ class Runner(object):
raise exc("Algorithm '%s' is not yet prepared" % self.name)
# Call the correct version of process()
if self.algorithm.is_analyzer:
if self.algorithm.isAnalyzer:
_check_argument(output, "output")
outputs_to_use = output
else:
_check_argument(outputs, "outputs")
outputs_to_use = outputs
if self.algorithm.type == Algorithm_.LEGACY:
if self.algorithm.type == Algorithm.LEGACY:
_check_argument(inputs, "inputs")
return loader.run(self.obj, "process", self.exc, inputs, outputs_to_use)
......@@ -505,41 +539,7 @@ class Runner(object):
# ----------------------------------------------------------
class Storage(utils.CodeStorage):
"""Resolves paths for algorithms
Parameters:
prefix (str): Establishes the prefix of your installation.
name (str): The name of the algorithm object in the format
``<user>/<name>/<version>``.
"""
asset_type = "algorithm"
asset_folder = "algorithms"
def __init__(self, prefix, name, language=None):
if name.count("/") != 2:
raise RuntimeError("invalid algorithm name: `%s'" % name)
self.username, self.name, self.version = name.split("/")
self.fullname = name
self.prefix = prefix
path = utils.hashed_or_simple(
self.prefix, self.asset_folder, name, suffix=".json"
)
path = path[:-5]
super(Storage, self).__init__(path, language)
# ----------------------------------------------------------
class Algorithm(Algorithm_):
class Algorithm(object):
"""Algorithms represent runnable components within the platform.
This class can only parse the meta-parameters of the algorithm (i.e., input
......@@ -613,6 +613,16 @@ class Algorithm(Algorithm_):
"""
LEGACY = "legacy"
SEQUENTIAL = "sequential"
AUTONOMOUS = "autonomous"
SEQUENTIAL_LOOP_EVALUATOR = "sequential_loop_evaluator"
AUTONOMOUS_LOOP_EVALUATOR = "autonomous_loop_evaluator"
SEQUENTIAL_LOOP_PROCESSOR = "sequential_loop_processor"
AUTONOMOUS_LOOP_PROCESSOR = "autonomous_loop_processor"
dataformat_klass = dataformat.DataFormat
def __init__(self, prefix, name, dataformat_cache=None, library_cache=None):
self._name = None
......@@ -848,6 +858,31 @@ class Algorithm(Algorithm_):
return self.data.get("type", Algorithm.SEQUENTIAL)
@property
def is_autonomous(self):
""" Returns whether the algorithm is in the autonomous category"""
return self.type in [
Algorithm.AUTONOMOUS,
Algorithm.AUTONOMOUS_LOOP_EVALUATOR,
Algorithm.AUTONOMOUS_LOOP_PROCESSOR,
]
@property
def is_sequential(self):
""" Returns whether the algorithm is in the sequential category"""
return self.type in [
Algorithm.SEQUENTIAL,
Algorithm.SEQUENTIAL_LOOP_EVALUATOR,
Algorithm.SEQUENTIAL_LOOP_PROCESSOR,
]
@property
def is_loop(self):
return self.type in [
Algorithm.SEQUENTIAL_LOOP_EVALUATOR,
Algorithm.AUTONOMOUS_LOOP_EVALUATOR,
]
@language.setter
def language(self, value):
"""Sets the current executable code programming language"""
......@@ -855,6 +890,73 @@ class Algorithm(Algorithm_):
self.storage.language = value
self.data["language"] = value
def clean_parameter(self, parameter, value):
"""Checks if a given value against a declared parameter
This method checks if the provided user value can be safe-cast to the
parameter type as defined on its specification and that it conforms to
any parameter-imposed restrictions.
Parameters:
parameter (str): The name of the parameter to check the value against
value (object): An object that will be safe cast into the defined
parameter type.
Returns:
The converted value, with an appropriate numpy type.
Raises:
KeyError: If the parameter cannot be found on this algorithm's
declaration.
ValueError: If the parameter cannot be safe cast into the algorithm's
type. Alternatively, a ``ValueError`` may also be raised if a range
or choice was specified and the value does not obey those settings
stipulated for the parameter
"""
if (self.parameters is None) or (parameter not in self.parameters):
raise KeyError(parameter)
retval = self.parameters[parameter]["type"].type(value)
if (
"choice" in self.parameters[parameter]
and retval not in self.parameters[parameter]["choice"]
):
raise ValueError(
"value for `%s' (%r) must be one of `[%s]'"
% (
parameter,
value,
", ".join(["%r" % k for k in self.parameters[parameter]["choice"]]),
)
)
if "range" in self.parameters[parameter] and (
retval < self.parameters[parameter]["range"][0]
or retval > self.parameters[parameter]["range"][1]
):
raise ValueError(
"value for `%s' (%r) must be picked within "
"interval `[%r, %r]'"
% (
parameter,
value,
self.parameters[parameter]["range"][0],
self.parameters[parameter]["range"][1],
)
)
return retval
@property
def valid(self):
"""A boolean that indicates if this algorithm is valid or not"""
......@@ -873,7 +975,7 @@ class Algorithm(Algorithm_):
return value
@property
def is_analyzer(self):
def isAnalyzer(self):
"""Returns whether this algorithms is an analyzer"""
return self.results is not None
......
......@@ -105,8 +105,9 @@ def setup_scalar(formatname, attrname, dtype, value, casting, add_defaults):
if value is None: # use the default for the type
return dtype.type()
else:
# zero is classified as int64 which can't be safely casted to uint64
if value:
if (
value
): # zero is classified as int64 which can't be safely casted to uint64
if not numpy.can_cast(numpy.array(value).dtype, dtype, casting=casting):
raise TypeError(
"cannot safely cast attribute `%s' on dataformat "
......@@ -121,11 +122,12 @@ def setup_scalar(formatname, attrname, dtype, value, casting, add_defaults):
else:
return str(value)
else: # it is a baseformat
else: # it is a dataformat
# check if the value is already a baseformat
if isinstance(value, baseformat):
return value
return dtype.from_dict(value, casting=casting, add_defaults=add_defaults)
return dtype().from_dict(value, casting=casting, add_defaults=add_defaults)
class _protected_str_ndarray(numpy.ndarray):
......@@ -251,7 +253,7 @@ def setup_array(formatname, attrname, shape, dtype, value, casting, add_defaults
def constructor(x):
"""Creates a data format base on the information provided by x"""
return dtype.from_dict(x, casting=casting, add_defaults=add_defaults)
return dtype().from_dict(x, casting=casting, add_defaults=add_defaults)
retval = numpy.frompyfunc(constructor, 1, 1)(retval).view(_protected_ndarray)
retval._format_dtype = dtype
......@@ -426,7 +428,8 @@ def unpack_scalar(dtype, fd):
return read_string(fd)[0]
else: # it is a dataformat
a = dtype.unpack_from(fd)
a = dtype()
a.unpack_from(fd)
return a
......@@ -434,7 +437,7 @@ class baseformat(object):
"""All dataformats are represented, in Python, by a derived class of this one
Construction is, by default, set to using a unsafe data type conversion.
For a 'safe' converter, use :py:meth:`baseformat.from_dict`, where you
For an 'safe' converter, use :py:meth:`baseformat.from_dict`, where you
can, optionally, set the casting style (see :py:func:`numpy.can_cast` for
details on the values this parameter can assume).
......@@ -445,14 +448,14 @@ class baseformat(object):
def __init__(self, **kwargs):
self.from_dict_(kwargs)
self.from_dict(kwargs, casting="unsafe", add_defaults=True)
@classmethod
def from_dict(cls, data, casting="safe", add_defaults=False):
"""Similar to initializing the object but allows you to change options
def from_dict(self, data, casting="safe", add_defaults=False):
"""Same as initializing the object, but with a less strict type casting
By default, casting is set to **safe** (see :py:func:`numpy.can_cast` for
details on the values this parameter can assume).
Construction is, by default, set to using a **unsafe** data type
conversion. See :py:func:`numpy.can_cast` for details on the values
this parameter can assume).
Parameters:
......@@ -467,16 +470,11 @@ class baseformat(object):
Use the constructor to get a default ``'unsafe'`` behaviour.
add_defaults (bool): If we should use defaults for missing
attributes. If this value is set to `True`, missing attributes
attributes. Incase this value is set to `True`, missing attributes
are set with defaults, otherwise, a :py:exc:`TypeError` is raise if
a missing attribute is found.
"""
return cls.__new__(cls).from_dict_(
data=data, casting=casting, add_defaults=add_defaults
)
def from_dict_(self, data, casting="safe", add_defaults=False):
if data is None:
data = {}
......@@ -486,22 +484,11 @@ class baseformat(object):
if not add_defaults:
# in this case, the user must provide all attributes
if user_attributes > declared_attributes:
extra_attributes = user_attributes - declared_attributes
raise AttributeError(
"Extra attributes (%s) for dataformat "
"`%s' were given which requires `%s'"
% (
", ".join(extra_attributes),
self._name,
", ".join(declared_attributes),
),
)
if user_attributes < declared_attributes:
if user_attributes != declared_attributes:
undeclared_attributes = declared_attributes - user_attributes
raise AttributeError(
"missing attributes (%s) for dataformat "
"`%s' which requires `%s'"
"`%s' which require `%s'"
% (
", ".join(undeclared_attributes),
self._name,
......@@ -581,14 +568,12 @@ class baseformat(object):
fd.close()
return retval
@classmethod
def unpack_from(cls, fd):
def unpack_from(self, fd):
"""Loads a binary representation of this object
We don't run any extra checks as an unpack operation is only supposed
to be carried out once the type compatibility has been established.
"""
self = cls.__new__(cls)
for key in sorted(self._format.keys()):
......@@ -604,15 +589,14 @@ class baseformat(object):
return self
@classmethod
def unpack(cls, s):
def unpack(self, s):
"""Loads a binary representation of this object from a string
Effectively, this method just calls :py:meth:`baseformat.unpack_from`
with a :py:data:`six.BytesIO` wrapped around the input string.
"""
return cls.unpack_from(six.BytesIO(s))
return self.unpack_from(six.BytesIO(s))
def isclose(self, other, *args, **kwargs):
"""Tests for closeness in the numerical sense.
......
......@@ -192,7 +192,7 @@ class DataFormat_:
``None``: Raises if an error occurs.
"""
klass = self.type
klass = self.type()
if isinstance(data, dict):
klass.from_dict(data, casting="safe", add_defaults=False)
elif isinstance(data, bytes):
......@@ -231,7 +231,10 @@ class DataFormat_:
# ----------------------------------------------------------
class DataFormat(DataFormat_):
# ----------------------------------------------------------
class DataFormat(object):
"""Data formats define the chunks of data that circulate between blocks.
Parameters:
......@@ -353,7 +356,7 @@ class DataFormat(DataFormat_):
def maybe_load_format(name, obj, dataformat_cache):
"""Tries to load a given dataformat from its relative path"""
if isinstance(obj, six.string_types) and "/" in obj: # load it
if isinstance(obj, six.string_types) and obj.find("/") != -1: # load it
if obj in dataformat_cache: # reuse
......@@ -424,11 +427,90 @@ class DataFormat(DataFormat_):
"""
return self.data.get("#extends")
@property
def type(self):
"""Returns a new type that can create instances of this dataformat.
The new returned type provides a basis to construct new objects which
represent the dataformat. It provides a simple JSON serializer and a
for-screen representation.
Example:
To create an object respecting the data format from a JSON
descriptor, use the following technique:
.. code-block:: python
ftype = dataformat(...).type
json = simplejson.loads(...)
newobj = ftype(**json) # instantiates the new object, checks format
To dump the object into JSON, use the following technique:
.. code-block:: python
simplejson.dumps(newobj.as_dict(), indent=4)
A string representation of the object uses the technique above to
pretty-print the object contents to the screen.
"""
if self.resolved is None:
raise RuntimeError(
"Cannot prototype while not properly initialized\n{}".format(
self.errors
)
)
classname = re.sub(r"[-/]", "_", self.name)
if not isinstance(classname, str):
classname = str(classname)
def init(self, **kwargs):
baseformat.__init__(self, **kwargs)
attributes = dict(__init__=init, _name=self.name, _format=self.resolved)
# create the converters for the class we're about to return
for k, v in self.resolved.items():
if isinstance(v, list): # it is an array
attributes[k] = copy.deepcopy(v)
if isinstance(v[-1], DataFormat):
attributes[k][-1] = v[-1].type
else:
if v[-1] in ("string", "str"):
attributes[k][-1] = str
else:
attributes[k][-1] = numpy.dtype(v[-1])
elif isinstance(v, DataFormat): # it is another dataformat
attributes[k] = v.type
else: # it is a simple type
if v in ("string", "str"):
attributes[k] = str
else:
attributes[k] = numpy.dtype(v)
return type(classname, (baseformat,), attributes)
@property
def valid(self):
"""A boolean that indicates if this dataformat is valid or not"""
return not bool(self.errors)
@property
def description(self):
"""The short description for this object"""
return self.data.get("#description", None)
@description.setter
def description(self, value):
"""Sets the short description for this object"""
self.data["#description"] = value
@property
def documentation(self):
"""The full-length description for this object"""
......
......@@ -3,17 +3,36 @@ import unittest
import numpy as np
# from ..algorithm import Algorithm_
from ..dataformat import DataFormat_
coordinates = DataFormat_(
definition={"x": int, "y": int, "name": str},
name="coordinates",
description="coordinates in an image",
)
sizes = DataFormat_(
definition={"width": int, "height": int, "array": [0, float]}, name="sizes"
)
rectangles = DataFormat_(
definition={"coords": coordinates, "size": sizes}, name="rectangles"
)
class CoordinatesToSizes:
def process(self, inputs, data_loaders, outputs):
# converts coordinates to sizes
coord = inputs["coordinates"]
new_size = sizes.type().from_dict(
dict(width=coord.x, height=coord.y, array=[coord.x, coord.y])
)
outputs["sizes"].write(new_size)
return True
class DataFormatTest(unittest.TestCase):
def test_dataformat_creation(self):
coordinates = DataFormat_(
definition={"x": int, "y": int, "name": str},
name="coordinates",
description="coordinates in an image",
)
def assert_hash(h, oracle):
self.assertEqual(h, oracle)
......@@ -27,7 +46,7 @@ class DataFormatTest(unittest.TestCase):
if oracle is None:
oracle = data
beat_type = data_format.type
beat_data = beat_type(**data)
beat_data = beat_type().from_dict(data)
obtained_data = beat_data.as_dict()
self.assertEqual(obtained_data, oracle)
# test validate
......@@ -50,24 +69,17 @@ class DataFormatTest(unittest.TestCase):
# test unsafe data conversion
with self.assertRaises(TypeError):
coordinates.type(x=1.0, y=2.0, name="test")
coordinates.type().from_dict(dict(x=1.0, y=2.0, name="test"))
# test extra data attributes
with self.assertRaises(AttributeError):
coordinates.type(x=1, y=2, name="test", extra="five")
coordinates.type().from_dict(dict(x=1, y=2, name="test", extra="five"))
# test missing data attributes
with self.assertRaises(AttributeError):
coordinates.type(x=1)
coordinates.type().from_dict(dict(x=1))
# test hierarchy
sizes = DataFormat_(
definition={"width": int, "height": int, "array": [0, float]}, name="sizes"
)
rectangles = DataFormat_(
definition={"coords": coordinates, "size": sizes}, name="rectangles"
)
data = dict(
coords=dict(x=1, y=2, name="test"), size=dict(width=3, height=4, array=[1])
)
......@@ -75,8 +87,10 @@ class DataFormatTest(unittest.TestCase):
# test when data is already in BEAT format
data_ = dict(
coords=coordinates.type(x=1, y=2, name="test"),
size=sizes.type(width=3, height=4, array=np.asarray([1], dtype="int32")),
coords=coordinates.type().from_dict(dict(x=1, y=2, name="test")),
size=sizes.type().from_dict(
dict(width=3, height=4, array=np.asarray([1], dtype="int32"))
),
)
assert_data_roundtrip(data_, rectangles, oracle=data)
......@@ -90,3 +104,15 @@ class DataFormatTest(unittest.TestCase):
rectangles.hash(),
"c59d4c08e57e5ddbeb698246eeb89d2a1b7a00892358a2dde4fa7ce1a588a690",
)
# class AlgorithmTest(unittest.TestCase):
# """docstring for AlgorithmTest"""
# def test_algorithm_creation(self):
# alg = CoordinatesToSizes()
# algorithm = Algorithm_(
# alg,
# {"main": {"inputs": ["coordinates"], "outputs": ["sizes"]}},
# type="sequential",
# )
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment