algorithm.py 33.1 KB
Newer Older
André Anjos's avatar
André Anjos committed
1
2
3
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

Samuel GAIST's avatar
Samuel GAIST committed
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
###################################################################################
#                                                                                 #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/               #
# Contact: beat.support@idiap.ch                                                  #
#                                                                                 #
# Redistribution and use in source and binary forms, with or without              #
# modification, are permitted provided that the following conditions are met:     #
#                                                                                 #
# 1. Redistributions of source code must retain the above copyright notice, this  #
# list of conditions and the following disclaimer.                                #
#                                                                                 #
# 2. Redistributions in binary form must reproduce the above copyright notice,    #
# this list of conditions and the following disclaimer in the documentation       #
# and/or other materials provided with the distribution.                          #
#                                                                                 #
# 3. Neither the name of the copyright holder nor the names of its contributors   #
# may be used to endorse or promote products derived from this software without   #
# specific prior written permission.                                              #
#                                                                                 #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED   #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE          #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE    #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL      #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR      #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER      #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,   #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE   #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.            #
#                                                                                 #
###################################################################################
André Anjos's avatar
André Anjos committed
35
36


37
38
39
40
41
42
43
"""
=========
algorithm
=========

Validation for algorithms
"""
André Anjos's avatar
André Anjos committed
44
45
46

import os
import sys
47
import logging
André Anjos's avatar
André Anjos committed
48
49
50
51
52
53
54
55

import six
import numpy
import simplejson

from . import dataformat
from . import library
from . import loader
56
57
58
from . import utils


59
60
61
logger = logging.getLogger(__name__)


62
# ----------------------------------------------------------
Philip ABBET's avatar
Philip ABBET committed
63

64
65

class Storage(utils.CodeStorage):
Philip ABBET's avatar
Philip ABBET committed
66
    """Resolves paths for algorithms
67

Philip ABBET's avatar
Philip ABBET committed
68
    Parameters:
69

70
      prefix (str): Establishes the prefix of your installation.
71

Philip ABBET's avatar
Philip ABBET committed
72
73
      name (str): The name of the algorithm object in the format
        ``<user>/<name>/<version>``.
74

Philip ABBET's avatar
Philip ABBET committed
75
    """
76

77
78
79
    asset_type = "algorithm"
    asset_folder = "algorithms"

Philip ABBET's avatar
Philip ABBET committed
80
    def __init__(self, prefix, name, language=None):
81

Samuel GAIST's avatar
Samuel GAIST committed
82
        if name.count("/") != 2:
Philip ABBET's avatar
Philip ABBET committed
83
            raise RuntimeError("invalid algorithm name: `%s'" % name)
84

Samuel GAIST's avatar
Samuel GAIST committed
85
        self.username, self.name, self.version = name.split("/")
Philip ABBET's avatar
Philip ABBET committed
86
        self.fullname = name
87
        self.prefix = prefix
88

89
90
91
        path = utils.hashed_or_simple(
            self.prefix, self.asset_folder, name, suffix=".json"
        )
92
        path = path[:-5]
Philip ABBET's avatar
Philip ABBET committed
93
        super(Storage, self).__init__(path, language)
94

André Anjos's avatar
André Anjos committed
95

96
# ----------------------------------------------------------
Philip ABBET's avatar
Philip ABBET committed
97

André Anjos's avatar
André Anjos committed
98
99

class Runner(object):
100
    """A special loader class for algorithms, with specialized methods
André Anjos's avatar
André Anjos committed
101

Philip ABBET's avatar
Philip ABBET committed
102
    Parameters:
André Anjos's avatar
André Anjos committed
103

104
105
      module (:std:term:`module`): The preloaded module containing the
        algorithm as returned by :py:func:`.loader.load_module`.
André Anjos's avatar
André Anjos committed
106

107
108
      obj_name (str): The name of the object within the module you're
        interested on
André Anjos's avatar
André Anjos committed
109

Philip ABBET's avatar
Philip ABBET committed
110
111
112
      algorithm (object): The algorithm instance that is used for parameter
        checking.

113
      exc (:std:term:`class`): The class to use as base exception when
114
        translating the exception from the user code. Read the documentation of
115
        :py:func:`.loader.run` for more details.
André Anjos's avatar
André Anjos committed
116

117
    """
André Anjos's avatar
André Anjos committed
118

119
    def __init__(self, module, obj_name, algorithm, exc=None):
André Anjos's avatar
André Anjos committed
120

Philip ABBET's avatar
Philip ABBET committed
121
122
        try:
            class_ = getattr(module, obj_name)
Samuel GAIST's avatar
Samuel GAIST committed
123
        except Exception:
Philip ABBET's avatar
Philip ABBET committed
124
125
126
127
            if exc is not None:
                type, value, traceback = sys.exc_info()
                six.reraise(exc, exc(value), traceback)
            else:
128
                raise  # Just re-raise the user exception
André Anjos's avatar
André Anjos committed
129

Samuel GAIST's avatar
Samuel GAIST committed
130
        self.obj = loader.run(class_, "__new__", exc)
Philip ABBET's avatar
Philip ABBET committed
131
132
133
        self.name = module.__name__
        self.algorithm = algorithm
        self.exc = exc
André Anjos's avatar
André Anjos committed
134

Samuel GAIST's avatar
Samuel GAIST committed
135
        self.ready = not hasattr(self.obj, "setup")
136
137

        has_api_v1 = self.algorithm.api_version == 1
Samuel GAIST's avatar
Samuel GAIST committed
138
        has_prepare = hasattr(self.obj, "prepare")
139
140
141
142
143
144

        self.prepared = has_api_v1 or not has_prepare

        if has_api_v1 and has_prepare:
            logger.warning("Prepare is a reserved function name starting with API V2")

Philip ABBET's avatar
Philip ABBET committed
145
146
    def _check_parameters(self, parameters):
        """Checks input parameters from the user and fill defaults"""
André Anjos's avatar
André Anjos committed
147

Philip ABBET's avatar
Philip ABBET committed
148
149
150
        user_keys = set(parameters.keys())
        algo_parameters = self.algorithm.parameters or {}
        valid_keys = set(algo_parameters.keys())
André Anjos's avatar
André Anjos committed
151

152
        # Checks the user is not trying to set an undeclared parameter
Philip ABBET's avatar
Philip ABBET committed
153
154
        if not user_keys.issubset(valid_keys):
            err_keys = user_keys - valid_keys
Samuel GAIST's avatar
Samuel GAIST committed
155
156
157
158
159
            message = (
                "parameters `%s' are not declared for algorithm `%s' - "
                "valid parameters are `%s'"
                % (",".join(err_keys), self.name, ",".join(valid_keys))
            )
Philip ABBET's avatar
Philip ABBET committed
160
161
            exc = self.exc or KeyError
            raise exc(message)
André Anjos's avatar
André Anjos committed
162

Philip ABBET's avatar
Philip ABBET committed
163
        retval = dict()
André Anjos's avatar
André Anjos committed
164

Philip ABBET's avatar
Philip ABBET committed
165
        for key, definition in algo_parameters.items():
André Anjos's avatar
André Anjos committed
166

Philip ABBET's avatar
Philip ABBET committed
167
168
169
170
            if key in parameters:
                try:
                    value = self.algorithm.clean_parameter(key, parameters[key])
                except Exception as e:
Samuel GAIST's avatar
Samuel GAIST committed
171
172
173
174
                    message = (
                        "parameter `%s' cannot be safely cast to the declared "
                        "type on algorithm `%s': %s" % (key, self.name, e)
                    )
Philip ABBET's avatar
Philip ABBET committed
175
176
                    exc = self.exc or ValueError
                    raise exc(message)
André Anjos's avatar
André Anjos committed
177

178
            else:  # User has not set a value, use the default
Samuel GAIST's avatar
Samuel GAIST committed
179
                value = algo_parameters[key]["default"]
André Anjos's avatar
André Anjos committed
180

181
            # In the end, set the value on the dictionary to be returned
Philip ABBET's avatar
Philip ABBET committed
182
            retval[key] = value
André Anjos's avatar
André Anjos committed
183

Philip ABBET's avatar
Philip ABBET committed
184
        return retval
André Anjos's avatar
André Anjos committed
185

186
    def setup(self, parameters):
187
        """Sets up the algorithm, only effective on the first call"""
André Anjos's avatar
André Anjos committed
188

189
        # Only effective on the first call
Philip ABBET's avatar
Philip ABBET committed
190
191
        if self.ready:
            return self.ready
André Anjos's avatar
André Anjos committed
192

Philip ABBET's avatar
Philip ABBET committed
193
        completed_parameters = self._check_parameters(parameters)
André Anjos's avatar
André Anjos committed
194

Samuel GAIST's avatar
Samuel GAIST committed
195
        self.ready = loader.run(self.obj, "setup", self.exc, completed_parameters)
Philip ABBET's avatar
Philip ABBET committed
196
197
        return self.ready

198
    def prepare(self, data_loaders):
199
        """Let the algorithm process the data on the non-principal channels"""
Philip ABBET's avatar
Philip ABBET committed
200

201
202
203
204
205
        # Only effective on the first call
        if self.prepared:
            return self.prepared

        # setup() must have run
Philip ABBET's avatar
Philip ABBET committed
206
207
        if not self.ready:
            exc = self.exc or RuntimeError
208
            raise exc("Algorithm '%s' is not yet setup" % self.name)
Philip ABBET's avatar
Philip ABBET committed
209

210
211
212
        # Not available in API version 1
        if self.algorithm.api_version == 1:
            self.prepared = True
Philip ABBET's avatar
Philip ABBET committed
213
            return True
André Anjos's avatar
André Anjos committed
214

215
        # The method is optional
Samuel GAIST's avatar
Samuel GAIST committed
216
        if hasattr(self.obj, "prepare"):
217
            if self.algorithm.type in [Algorithm.AUTONOMOUS, Algorithm.LOOP_USER]:
Samuel GAIST's avatar
Samuel GAIST committed
218
219
220
                self.prepared = loader.run(
                    self.obj, "prepare", self.exc, data_loaders.secondaries()
                )
221
            else:
Samuel GAIST's avatar
Samuel GAIST committed
222
                self.prepared = loader.run(self.obj, "prepare", self.exc, data_loaders)
223
224
        else:
            self.prepared = True
Philip ABBET's avatar
Philip ABBET committed
225

226
        return self.prepared
André Anjos's avatar
André Anjos committed
227

Samuel GAIST's avatar
Samuel GAIST committed
228
229
230
231
232
233
234
235
    def process(
        self,
        inputs=None,
        data_loaders=None,
        outputs=None,
        output=None,
        loop_channel=None,
    ):
236
        """Runs through data"""
André Anjos's avatar
André Anjos committed
237

238
239
240
241
        exc = self.exc or RuntimeError

        def _check_argument(argument, name):
            if argument is None:
Samuel GAIST's avatar
Samuel GAIST committed
242
                raise exc("Missing argument: %s" % name)
243
244

        # setup() must have run
Philip ABBET's avatar
Philip ABBET committed
245
        if not self.ready:
246
            raise exc("Algorithm '%s' is not yet setup" % self.name)
André Anjos's avatar
André Anjos committed
247

248
249
250
251
252
253
        # prepare() must have run
        if not self.prepared:
            raise exc("Algorithm '%s' is not yet prepared" % self.name)

        # Call the correct version of process()
        if self.algorithm.isAnalyzer:
Samuel GAIST's avatar
Samuel GAIST committed
254
            _check_argument(output, "output")
255
256
            outputs_to_use = output
        else:
Samuel GAIST's avatar
Samuel GAIST committed
257
            _check_argument(outputs, "outputs")
258
259
260
            outputs_to_use = outputs

        if self.algorithm.type == Algorithm.LEGACY:
Samuel GAIST's avatar
Samuel GAIST committed
261
262
            _check_argument(inputs, "inputs")
            return loader.run(self.obj, "process", self.exc, inputs, outputs_to_use)
263
264

        else:
Samuel GAIST's avatar
Samuel GAIST committed
265
            _check_argument(data_loaders, "data_loaders")
266
267

            if self.algorithm.type == Algorithm.SEQUENTIAL:
Samuel GAIST's avatar
Samuel GAIST committed
268
                _check_argument(inputs, "inputs")
269

Samuel GAIST's avatar
Samuel GAIST committed
270
271
272
                return loader.run(
                    self.obj, "process", self.exc, inputs, data_loaders, outputs_to_use
                )
273

274
            elif self.algorithm.is_autonomous:
Samuel GAIST's avatar
Samuel GAIST committed
275
                run_args = [self.obj, "process", self.exc, data_loaders, outputs_to_use]
276

Samuel GAIST's avatar
Samuel GAIST committed
277
                has_loop_arg = utils.has_argument(self.obj.process, "loop_channel")
278
                if loop_channel is not None:
279
                    if has_loop_arg:
280
281
                        run_args.append(loop_channel)
                    else:
Samuel GAIST's avatar
Samuel GAIST committed
282
283
284
285
                        raise exc(
                            "Algorithm '%s' is not a valid loop enabled algorithm"
                            % self.name
                        )
286
                elif has_loop_arg:
Samuel GAIST's avatar
Samuel GAIST committed
287
288
289
290
                    raise exc(
                        "Algorithm '%s' is a loop enabled algorithm but no loop_channel given"
                        % self.name
                    )
291
292

                return loader.run(*run_args)
293
294

            else:
Samuel GAIST's avatar
Samuel GAIST committed
295
                raise exc("Unknown algorithm type: %s" % self.algorithm.type)
André Anjos's avatar
André Anjos committed
296

297
298
299
300
301
302
    def validate(self, result):
        """Validates the given results"""

        exc = self.exc or RuntimeError

        if self.algorithm.type != Algorithm.LOOP:
Samuel GAIST's avatar
Samuel GAIST committed
303
            raise exc("Wrong algorithm type: %s" % self.algorithm.type)
304
305
306
307
308
309
310
311
312

        # setup() must have run
        if not self.ready:
            raise exc("Algorithm '%s' is not yet setup" % self.name)

        # prepare() must have run
        if not self.prepared:
            raise exc("Algorithm '%s' is not yet prepared" % self.name)

Samuel GAIST's avatar
Samuel GAIST committed
313
        answer = loader.run(self.obj, "validate", self.exc, result)
314
315
316
317
318

        if not isinstance(answer, tuple):
            raise exc("Improper implementation: validate must return a tuple")

        return answer
319

Philip ABBET's avatar
Philip ABBET committed
320
    def __getattr__(self, key):
321
322
        """Returns an attribute of the algorithm - only called at last resort
        """
Philip ABBET's avatar
Philip ABBET committed
323
        return getattr(self.obj, key)
André Anjos's avatar
André Anjos committed
324
325


326
# ----------------------------------------------------------
Philip ABBET's avatar
Philip ABBET committed
327

328

André Anjos's avatar
André Anjos committed
329
class Algorithm(object):
Philip ABBET's avatar
Philip ABBET committed
330
    """Algorithms represent runnable components within the platform.
André Anjos's avatar
André Anjos committed
331

Philip ABBET's avatar
Philip ABBET committed
332
333
    This class can only parse the meta-parameters of the algorithm (i.e., input
    and output declaration, grouping, synchronization details, parameters and
334
335
    splittability). The actual algorithm is not directly treated by this class.
    It can, however, provide you with a loader for actually running the
Philip ABBET's avatar
Philip ABBET committed
336
    algorithmic code (see :py:meth:`Algorithm.runner`).
André Anjos's avatar
André Anjos committed
337
338


Philip ABBET's avatar
Philip ABBET committed
339
    Parameters:
André Anjos's avatar
André Anjos committed
340

341
      prefix (str): Establishes the prefix of your installation.
André Anjos's avatar
André Anjos committed
342

Philip ABBET's avatar
Philip ABBET committed
343
      name (str): The fully qualified algorithm name (e.g. ``user/algo/1``)
André Anjos's avatar
André Anjos committed
344

345
346
347
348
      dataformat_cache (:py:class:`dict`, Optional): A dictionary mapping
        dataformat names to loaded dataformats. This parameter is optional and,
        if passed, may greatly speed-up algorithm loading times as dataformats
        that are already loaded may be re-used.
André Anjos's avatar
André Anjos committed
349

350
351
352
353
      library_cache (:py:class:`dict`, Optional): A dictionary mapping library
        names to loaded libraries. This parameter is optional and, if passed,
        may greatly speed-up library loading times as libraries that are
        already loaded may be re-used.
André Anjos's avatar
André Anjos committed
354
355


Philip ABBET's avatar
Philip ABBET committed
356
    Attributes:
André Anjos's avatar
André Anjos committed
357

Philip ABBET's avatar
Philip ABBET committed
358
      name (str): The algorithm name
André Anjos's avatar
André Anjos committed
359

360
361
      dataformats (dict): A dictionary containing all pre-loaded dataformats
        used by this algorithm. Data format objects will be of type
362
        :py:class:`.dataformat.DataFormat`.
André Anjos's avatar
André Anjos committed
363

364
365
      libraries (dict): A mapping object defining other libraries this
        algorithm needs to load so it can work properly.
André Anjos's avatar
André Anjos committed
366

Philip ABBET's avatar
Philip ABBET committed
367
368
      uses (dict): A mapping object defining the required library import name
        (keys) and the full-names (values).
André Anjos's avatar
André Anjos committed
369

370
371
      parameters (dict): A dictionary containing all pre-defined parameters
        that this algorithm accepts.
André Anjos's avatar
André Anjos committed
372

Philip ABBET's avatar
Philip ABBET committed
373
374
      splittable (bool): A boolean value that indicates if this algorithm is
        automatically parallelizeable by our backend.
André Anjos's avatar
André Anjos committed
375

Philip ABBET's avatar
Philip ABBET committed
376
      input_map (dict): A dictionary where the key is the input name and the
377
378
        value, its type. All input names (potentially from different groups)
        are comprised in this dictionary.
André Anjos's avatar
André Anjos committed
379

Philip ABBET's avatar
Philip ABBET committed
380
      output_map (dict): A dictionary where the key is the output name and the
381
382
        value, its type. All output names (potentially from different groups)
        are comprised in this dictionary.
André Anjos's avatar
André Anjos committed
383

384
385
386
387
      results (dict): If this algorithm is actually an analyzer (i.e., there
        are no formal outputs, but results that must be saved by the platform),
        then this dictionary contains the names and data types of those
        elements.
André Anjos's avatar
André Anjos committed
388

Philip ABBET's avatar
Philip ABBET committed
389
390
      groups (dict): A list containing dictionaries with inputs and outputs
        belonging to the same synchronization group.
André Anjos's avatar
André Anjos committed
391

Philip ABBET's avatar
Philip ABBET committed
392
393
      errors (list): A list containing errors found while loading this
        algorithm.
394

Philip ABBET's avatar
Philip ABBET committed
395
396
      data (dict): The original data for this algorithm, as loaded by our JSON
        decoder.
André Anjos's avatar
André Anjos committed
397

Philip ABBET's avatar
Philip ABBET committed
398
399
      code (str): The code that is associated with this algorithm, loaded as a
        text (or binary) file.
André Anjos's avatar
André Anjos committed
400

Philip ABBET's avatar
Philip ABBET committed
401
    """
André Anjos's avatar
André Anjos committed
402

Samuel GAIST's avatar
Samuel GAIST committed
403
404
405
406
407
    LEGACY = "legacy"
    SEQUENTIAL = "sequential"
    AUTONOMOUS = "autonomous"
    LOOP = "loop"
    LOOP_USER = "loop_user"
Philip ABBET's avatar
Philip ABBET committed
408

Philip ABBET's avatar
Philip ABBET committed
409
    def __init__(self, prefix, name, dataformat_cache=None, library_cache=None):
André Anjos's avatar
André Anjos committed
410

Philip ABBET's avatar
Philip ABBET committed
411
412
413
414
415
416
417
        self._name = None
        self.storage = None
        self.prefix = prefix
        self.dataformats = {}
        self.libraries = {}
        self.groups = []
        self.errors = []
André Anjos's avatar
André Anjos committed
418

Philip ABBET's avatar
Philip ABBET committed
419
420
        dataformat_cache = dataformat_cache if dataformat_cache is not None else {}
        library_cache = library_cache if library_cache is not None else {}
André Anjos's avatar
André Anjos committed
421

Philip ABBET's avatar
Philip ABBET committed
422
        self._load(name, dataformat_cache, library_cache)
André Anjos's avatar
André Anjos committed
423

Philip ABBET's avatar
Philip ABBET committed
424
425
    def _load(self, data, dataformat_cache, library_cache):
        """Loads the algorithm"""
426

Philip ABBET's avatar
Philip ABBET committed
427
        self._name = data
428

Philip ABBET's avatar
Philip ABBET committed
429
430
431
        self.storage = Storage(self.prefix, data)
        json_path = self.storage.json.path
        if not self.storage.exists():
Samuel GAIST's avatar
Samuel GAIST committed
432
            self.errors.append("Algorithm declaration file not found: %s" % json_path)
Philip ABBET's avatar
Philip ABBET committed
433
            return
434

Samuel GAIST's avatar
Samuel GAIST committed
435
436
        with open(json_path, "rb") as f:
            self.data = simplejson.loads(f.read().decode("utf-8"))
437

Philip ABBET's avatar
Philip ABBET committed
438
        self.code_path = self.storage.code.path
439
        self.code = self.storage.code.load()
André Anjos's avatar
André Anjos committed
440

Samuel GAIST's avatar
Samuel GAIST committed
441
        self.groups = self.data["groups"]
André Anjos's avatar
André Anjos committed
442

Philip ABBET's avatar
Philip ABBET committed
443
        # create maps for easy access to data
Samuel GAIST's avatar
Samuel GAIST committed
444
445
446
447
448
449
450
451
452
453
454
455
456
        self.input_map = dict(
            [(k, v["type"]) for g in self.groups for k, v in g["inputs"].items()]
        )
        self.output_map = dict(
            [
                (k, v["type"])
                for g in self.groups
                for k, v in g.get("outputs", {}).items()
            ]
        )
        self.loop_map = dict(
            [(k, v["type"]) for g in self.groups for k, v in g.get("loop", {}).items()]
        )
André Anjos's avatar
André Anjos committed
457

Philip ABBET's avatar
Philip ABBET committed
458
459
460
        self._load_dataformats(dataformat_cache)
        self._convert_parameter_types()
        self._load_libraries(library_cache)
André Anjos's avatar
André Anjos committed
461

Philip ABBET's avatar
Philip ABBET committed
462
463
464
    def _load_dataformats(self, dataformat_cache):
        """Makes sure we can load all requested formats
        """
André Anjos's avatar
André Anjos committed
465

Philip ABBET's avatar
Philip ABBET committed
466
        for group in self.groups:
André Anjos's avatar
André Anjos committed
467

Samuel GAIST's avatar
Samuel GAIST committed
468
469
470
            for name, input in group["inputs"].items():
                if input["type"] in self.dataformats:
                    continue
André Anjos's avatar
André Anjos committed
471

Samuel GAIST's avatar
Samuel GAIST committed
472
473
                if dataformat_cache and input["type"] in dataformat_cache:  # reuse
                    thisformat = dataformat_cache[input["type"]]
474
                else:  # load it
Samuel GAIST's avatar
Samuel GAIST committed
475
                    thisformat = dataformat.DataFormat(self.prefix, input["type"])
476
                    if dataformat_cache is not None:  # update it
Samuel GAIST's avatar
Samuel GAIST committed
477
                        dataformat_cache[input["type"]] = thisformat
André Anjos's avatar
André Anjos committed
478

Samuel GAIST's avatar
Samuel GAIST committed
479
                self.dataformats[input["type"]] = thisformat
André Anjos's avatar
André Anjos committed
480

Samuel GAIST's avatar
Samuel GAIST committed
481
482
            if "outputs" not in group:
                continue
André Anjos's avatar
André Anjos committed
483

Samuel GAIST's avatar
Samuel GAIST committed
484
485
486
            for name, output in group["outputs"].items():
                if output["type"] in self.dataformats:
                    continue
André Anjos's avatar
André Anjos committed
487

Samuel GAIST's avatar
Samuel GAIST committed
488
489
                if dataformat_cache and output["type"] in dataformat_cache:  # reuse
                    thisformat = dataformat_cache[output["type"]]
490
                else:  # load it
Samuel GAIST's avatar
Samuel GAIST committed
491
492
493
                    thisformat = dataformat.DataFormat(self.prefix, output["type"])
                    if dataformat_cache is not None:  # update it
                        dataformat_cache[output["type"]] = thisformat
André Anjos's avatar
André Anjos committed
494

Samuel GAIST's avatar
Samuel GAIST committed
495
                self.dataformats[output["type"]] = thisformat
André Anjos's avatar
André Anjos committed
496

Samuel GAIST's avatar
Samuel GAIST committed
497
            if "loop" not in group:
498
499
                continue

Samuel GAIST's avatar
Samuel GAIST committed
500
501
            for name, entry in group["loop"].items():
                entry_format = entry["type"]
502
503
504
505
506
507
508
509
510
511
512
513
                if entry_format in self.dataformats:
                    continue

                if dataformat_cache and entry_format in dataformat_cache:
                    thisformat = dataformat_cache[entry_format]
                else:
                    thisformat = dataformat.DataFormat(self.prefix, entry_format)
                    if dataformat_cache is not None:
                        dataformat_cache[entry_format] = thisformat

                self.dataformats[entry_format] = thisformat

Philip ABBET's avatar
Philip ABBET committed
514
        if self.results:
André Anjos's avatar
André Anjos committed
515

Philip ABBET's avatar
Philip ABBET committed
516
            for name, result in self.results.items():
André Anjos's avatar
André Anjos committed
517

Samuel GAIST's avatar
Samuel GAIST committed
518
                if result["type"].find("/") != -1:
André Anjos's avatar
André Anjos committed
519

Samuel GAIST's avatar
Samuel GAIST committed
520
521
                    if result["type"] in self.dataformats:
                        continue
André Anjos's avatar
André Anjos committed
522

Samuel GAIST's avatar
Samuel GAIST committed
523
524
                    if dataformat_cache and result["type"] in dataformat_cache:  # reuse
                        thisformat = dataformat_cache[result["type"]]
Philip ABBET's avatar
Philip ABBET committed
525
                    else:
Samuel GAIST's avatar
Samuel GAIST committed
526
527
528
                        thisformat = dataformat.DataFormat(self.prefix, result["type"])
                        if dataformat_cache is not None:  # update it
                            dataformat_cache[result["type"]] = thisformat
André Anjos's avatar
André Anjos committed
529

Samuel GAIST's avatar
Samuel GAIST committed
530
                    self.dataformats[result["type"]] = thisformat
André Anjos's avatar
André Anjos committed
531

Philip ABBET's avatar
Philip ABBET committed
532
533
534
    def _convert_parameter_types(self):
        """Converts types to numpy equivalents, checks defaults, ranges and choices
        """
André Anjos's avatar
André Anjos committed
535

Philip ABBET's avatar
Philip ABBET committed
536
537
538
539
        def _try_convert(name, tp, value, desc):
            try:
                return tp.type(value)
            except Exception as e:
Samuel GAIST's avatar
Samuel GAIST committed
540
541
542
543
                self.errors.append(
                    "%s for parameter `%s' cannot be cast to type "
                    "`%s': %s" % (desc, name, tp.name, e)
                )
André Anjos's avatar
André Anjos committed
544

Samuel GAIST's avatar
Samuel GAIST committed
545
546
        if self.parameters is None:
            return
André Anjos's avatar
André Anjos committed
547

Philip ABBET's avatar
Philip ABBET committed
548
        for name, parameter in self.parameters.items():
Samuel GAIST's avatar
Samuel GAIST committed
549
550
            if parameter["type"] == "string":
                parameter["type"] = numpy.dtype("str")
Philip ABBET's avatar
Philip ABBET committed
551
            else:
Samuel GAIST's avatar
Samuel GAIST committed
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
                parameter["type"] = numpy.dtype(parameter["type"])

            if "range" in parameter:
                parameter["range"][0] = _try_convert(
                    name, parameter["type"], parameter["range"][0], "start of range"
                )
                parameter["range"][1] = _try_convert(
                    name, parameter["type"], parameter["range"][1], "end of range"
                )
                if parameter["range"][0] >= parameter["range"][1]:
                    self.errors.append(
                        "range for parameter `%s' has a start greater "
                        "then the end value (%r >= %r)"
                        % (name, parameter["range"][0], parameter["range"][1])
                    )

            if "choice" in parameter:
                for i, choice in enumerate(parameter["choice"]):
                    parameter["choice"][i] = _try_convert(
                        name,
                        parameter["type"],
                        parameter["choice"][i],
                        "choice[%d]" % i,
                    )

            if "default" in parameter:
                parameter["default"] = _try_convert(
                    name, parameter["type"], parameter["default"], "default"
                )

                if "range" in parameter:  # check range
                    if (
                        parameter["default"] < parameter["range"][0]
                        or parameter["default"] > parameter["range"][1]
                    ):
                        self.errors.append(
                            "default for parameter `%s' (%r) is not "
                            "within parameter range [%r, %r]"
                            % (
                                name,
                                parameter["default"],
                                parameter["range"][0],
                                parameter["range"][1],
                            )
                        )
André Anjos's avatar
André Anjos committed
597

Samuel GAIST's avatar
Samuel GAIST committed
598
599
600
601
602
603
604
605
606
607
608
                if "choice" in parameter:  # check choices
                    if parameter["default"] not in parameter["choice"]:
                        self.errors.append(
                            "default for parameter `%s' (%r) is not "
                            "a valid choice `[%s]'"
                            % (
                                name,
                                parameter["default"],
                                ", ".join(["%r" % k for k in parameter["choice"]]),
                            )
                        )
André Anjos's avatar
André Anjos committed
609

Philip ABBET's avatar
Philip ABBET committed
610
    def _load_libraries(self, library_cache):
André Anjos's avatar
André Anjos committed
611

Philip ABBET's avatar
Philip ABBET committed
612
        # all used libraries must be loadable; cannot use self as a library
André Anjos's avatar
André Anjos committed
613

Philip ABBET's avatar
Philip ABBET committed
614
        if self.uses:
André Anjos's avatar
André Anjos committed
615

Philip ABBET's avatar
Philip ABBET committed
616
            for name, value in self.uses.items():
André Anjos's avatar
André Anjos committed
617

Samuel GAIST's avatar
Samuel GAIST committed
618
619
620
                self.libraries[value] = library_cache.setdefault(
                    value, library.Library(self.prefix, value, library_cache)
                )
André Anjos's avatar
André Anjos committed
621

Philip ABBET's avatar
Philip ABBET committed
622
623
624
625
    @property
    def name(self):
        """Returns the name of this object
        """
Samuel GAIST's avatar
Samuel GAIST committed
626
        return self._name or "__unnamed_algorithm__"
627

Philip ABBET's avatar
Philip ABBET committed
628
629
    @name.setter
    def name(self, value):
630
631
632
633
634
        """Sets the name of this object

        Parameters:
            name (str): Name of this object
        """
635

Samuel GAIST's avatar
Samuel GAIST committed
636
        if self.data["language"] == "unknown":
Philip ABBET's avatar
Philip ABBET committed
637
            raise RuntimeError("algorithm has no programming language set")
638

Philip ABBET's avatar
Philip ABBET committed
639
        self._name = value
Samuel GAIST's avatar
Samuel GAIST committed
640
        self.storage = Storage(self.prefix, value, self.data["language"])
André Anjos's avatar
André Anjos committed
641

Philip ABBET's avatar
Philip ABBET committed
642
643
644
    @property
    def schema_version(self):
        """Returns the schema version"""
Samuel GAIST's avatar
Samuel GAIST committed
645
        return self.data.get("schema_version", 1)
André Anjos's avatar
André Anjos committed
646

Philip ABBET's avatar
Philip ABBET committed
647
648
649
    @property
    def language(self):
        """Returns the current language set for the executable code"""
Samuel GAIST's avatar
Samuel GAIST committed
650
        return self.data["language"]
651

Philip ABBET's avatar
Philip ABBET committed
652
653
654
    @property
    def api_version(self):
        """Returns the API version"""
Samuel GAIST's avatar
Samuel GAIST committed
655
        return self.data.get("api_version", 1)
Philip ABBET's avatar
Philip ABBET committed
656
657
658
659
660
661
662

    @property
    def type(self):
        """Returns the type of algorithm"""
        if self.api_version == 1:
            return Algorithm.LEGACY

Samuel GAIST's avatar
Samuel GAIST committed
663
        return self.data.get("type", Algorithm.SEQUENTIAL)
Philip ABBET's avatar
Philip ABBET committed
664

665
666
667
    @property
    def is_autonomous(self):
        """ Returns whether the algorithm is in the autonomous category"""
Samuel GAIST's avatar
Samuel GAIST committed
668
        return self.type in [Algorithm.AUTONOMOUS, Algorithm.LOOP_USER, Algorithm.LOOP]
669

Philip ABBET's avatar
Philip ABBET committed
670
671
672
673
674
    @language.setter
    def language(self, value):
        """Sets the current executable code programming language"""
        if self.storage:
            self.storage.language = value
Samuel GAIST's avatar
Samuel GAIST committed
675
        self.data["language"] = value
676

Philip ABBET's avatar
Philip ABBET committed
677
678
    def clean_parameter(self, parameter, value):
        """Checks if a given value against a declared parameter
André Anjos's avatar
André Anjos committed
679

Philip ABBET's avatar
Philip ABBET committed
680
        This method checks if the provided user value can be safe-cast to the
681
682
        parameter type as defined on its specification and that it conforms to
        any parameter-imposed restrictions.
André Anjos's avatar
André Anjos committed
683
684


Philip ABBET's avatar
Philip ABBET committed
685
        Parameters:
André Anjos's avatar
André Anjos committed
686

Philip ABBET's avatar
Philip ABBET committed
687
          parameter (str): The name of the parameter to check the value against
André Anjos's avatar
André Anjos committed
688

Philip ABBET's avatar
Philip ABBET committed
689
690
          value (object): An object that will be safe cast into the defined
            parameter type.
André Anjos's avatar
André Anjos committed
691
692


Philip ABBET's avatar
Philip ABBET committed
693
        Returns:
André Anjos's avatar
André Anjos committed
694

Philip ABBET's avatar
Philip ABBET committed
695
          The converted value, with an appropriate numpy type.
André Anjos's avatar
André Anjos committed
696
697


Philip ABBET's avatar
Philip ABBET committed
698
        Raises:
André Anjos's avatar
André Anjos committed
699

Philip ABBET's avatar
Philip ABBET committed
700
701
          KeyError: If the parameter cannot be found on this algorithm's
            declaration.
André Anjos's avatar
André Anjos committed
702

Philip ABBET's avatar
Philip ABBET committed
703
          ValueError: If the parameter cannot be safe cast into the algorithm's
704
705
            type. Alternatively, a ``ValueError`` may also be raised if a range
            or choice was specified and the value does not obey those settings
Philip ABBET's avatar
Philip ABBET committed
706
707
            stipulated for the parameter
        """
André Anjos's avatar
André Anjos committed
708

Philip ABBET's avatar
Philip ABBET committed
709
710
        if (self.parameters is None) or (parameter not in self.parameters):
            raise KeyError(parameter)
André Anjos's avatar
André Anjos committed
711

Samuel GAIST's avatar
Samuel GAIST committed
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
        retval = self.parameters[parameter]["type"].type(value)

        if (
            "choice" in self.parameters[parameter]
            and retval not in self.parameters[parameter]["choice"]
        ):
            raise ValueError(
                "value for `%s' (%r) must be one of `[%s]'"
                % (
                    parameter,
                    value,
                    ", ".join(["%r" % k for k in self.parameters[parameter]["choice"]]),
                )
            )

        if "range" in self.parameters[parameter] and (
            retval < self.parameters[parameter]["range"][0]
            or retval > self.parameters[parameter]["range"][1]
        ):
            raise ValueError(
                "value for `%s' (%r) must be picked within "
                "interval `[%r, %r]'"
                % (
                    parameter,
                    value,
                    self.parameters[parameter]["range"][0],
                    self.parameters[parameter]["range"][1],
                )
            )
André Anjos's avatar
André Anjos committed
741

Philip ABBET's avatar
Philip ABBET committed
742
        return retval
André Anjos's avatar
André Anjos committed
743

Philip ABBET's avatar
Philip ABBET committed
744
745
746
    @property
    def valid(self):
        """A boolean that indicates if this algorithm is valid or not"""
747

Philip ABBET's avatar
Philip ABBET committed
748
        return not bool(self.errors)
749

Philip ABBET's avatar
Philip ABBET committed
750
751
    @property
    def uses(self):
Samuel GAIST's avatar
Samuel GAIST committed
752
        return self.data.get("uses")
André Anjos's avatar
André Anjos committed
753

Philip ABBET's avatar
Philip ABBET committed
754
755
    @uses.setter
    def uses(self, value):
Samuel GAIST's avatar
Samuel GAIST committed
756
        self.data["uses"] = value
Philip ABBET's avatar
Philip ABBET committed
757
        return value
758

759
760
    @property
    def isAnalyzer(self):
761
762
        """Returns whether this algorithms is an analyzer"""

Samuel GAIST's avatar
Samuel GAIST committed
763
        return self.results is not None
764

Philip ABBET's avatar
Philip ABBET committed
765
766
    @property
    def results(self):
767
768
        """The results of this algorithm"""

Samuel GAIST's avatar
Samuel GAIST committed
769
        return self.data.get("results")
André Anjos's avatar
André Anjos committed
770

Philip ABBET's avatar
Philip ABBET committed
771
772
    @results.setter
    def results(self, value):
Samuel GAIST's avatar
Samuel GAIST committed
773
        self.data["results"] = value
Philip ABBET's avatar
Philip ABBET committed
774
        return value
775

Philip ABBET's avatar
Philip ABBET committed
776
777
    @property
    def parameters(self):
778
779
        """The parameters of this algorithm"""

Samuel GAIST's avatar
Samuel GAIST committed
780
        return self.data.get("parameters")
André Anjos's avatar
André Anjos committed
781

Philip ABBET's avatar
Philip ABBET committed
782
783
    @parameters.setter
    def parameters(self, value):
Samuel GAIST's avatar
Samuel GAIST committed
784
        self.data["parameters"] = value
Philip ABBET's avatar
Philip ABBET committed
785
        return value
786

Philip ABBET's avatar
Philip ABBET committed
787
788
    @property
    def splittable(self):
789
        """Whether this algorithm can be split between several processes"""
Samuel GAIST's avatar
Samuel GAIST committed
790
        return self.data.get("splittable", False)
André Anjos's avatar
André Anjos committed
791

Philip ABBET's avatar
Philip ABBET committed
792
793
    @splittable.setter
    def splittable(self, value):
Samuel GAIST's avatar
Samuel GAIST committed
794
        self.data["splittable"] = value
Philip ABBET's avatar
Philip ABBET committed
795
        return value
796

797
798
799
    @property
    def description(self):
        """The short description for this object"""
Samuel GAIST's avatar
Samuel GAIST committed
800
        return self.data.get("description", None)
801
802
803
804

    @description.setter
    def description(self, value):
        """Sets the short description for this object"""
Samuel GAIST's avatar
Samuel GAIST committed
805
        self.data["description"] = value
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824

    @property
    def documentation(self):
        """The full-length description for this object"""

        if not self._name:
            raise RuntimeError("algorithm has no name")

        if self.storage.doc.exists():
            return self.storage.doc.load()
        return None

    @documentation.setter
    def documentation(self, value):
        """Sets the full-length description for this object"""

        if not self._name:
            raise RuntimeError("algorithm has no name")

Samuel GAIST's avatar
Samuel GAIST committed
825
        if hasattr(value, "read"):
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
            self.storage.doc.save(value.read())
        else:
            self.storage.doc.save(value)

    def hash(self):
        """Returns the hexadecimal hash for the current algorithm"""

        if not self._name:
            raise RuntimeError("algorithm has no name")

        return self.storage.hash()

    def result_dataformat(self):
        """Generates, on-the-fly, the dataformat for the result readout"""

        if not self.results:
Samuel GAIST's avatar
Samuel GAIST committed
842
843
844
            raise TypeError(
                "algorithm `%s' is a block algorithm, not an analyzer" % (self.name)
            )
845

Samuel GAIST's avatar
Samuel GAIST committed
846
847
848
        format = dataformat.DataFormat(
            self.prefix, dict([(k, v["type"]) for k, v in self.results.items()])
        )
849

Samuel GAIST's avatar
Samuel GAIST committed
850
        format.name = "analysis:" + self.name
851
852
853

        return format

Philip ABBET's avatar
Philip ABBET committed
854
855
    def uses_dict(self):
        """Returns the usage dictionary for all dependent modules"""
André Anjos's avatar
André Anjos committed
856

Samuel GAIST's avatar
Samuel GAIST committed
857
        if self.data["language"] == "unknown":
Philip ABBET's avatar
Philip ABBET committed
858
            raise RuntimeError("algorithm has no programming language set")
859

Philip ABBET's avatar
Philip ABBET committed
860
861
        if not self._name:
            raise RuntimeError("algorithm has no name")
862

Philip ABBET's avatar
Philip ABBET committed
863
        retval = {}
André Anjos's avatar
André Anjos committed
864

Philip ABBET's avatar
Philip ABBET committed
865
866
867
        if self.uses is not None:
            for name, value in self.uses.items():
                retval[name] = dict(
Samuel GAIST's avatar
Samuel GAIST committed
868
869
870
                    path=self.libraries[value].storage.code.path,
                    uses=self.libraries[value].uses_dict(),
                )
André Anjos's avatar
André Anjos committed
871

Philip ABBET's avatar
Philip ABBET committed
872
        return retval
André Anjos's avatar
André Anjos committed
873

Samuel GAIST's avatar
Samuel GAIST committed
874
    def runner(self, klass="Algorithm", exc=None):
Philip ABBET's avatar
Philip ABBET committed
875
        """Returns a runnable algorithm object.
André Anjos's avatar
André Anjos committed
876

Philip ABBET's avatar
Philip ABBET committed
877
        Parameters:
André Anjos's avatar
André Anjos committed
878

Philip ABBET's avatar
Philip ABBET committed
879
          klass (str): The name of the class to load the runnable algorithm from
André Anjos's avatar
André Anjos committed
880

881
882
          exc (:std:term:`class`): If passed, must be a valid exception class
            that will be used to report errors in the read-out of this algorithm's code.
André Anjos's avatar
André Anjos committed
883

Philip ABBET's avatar
Philip ABBET committed
884
        Returns:
André Anjos's avatar
André Anjos committed
885

886
          :py:class:`Runner`: An instance of the algorithm,
Philip ABBET's avatar
Philip ABBET committed
887
888
889
            which will be constructed, but not setup.  You **must** set it up
            before using the ``process`` method.
        """
André Anjos's avatar
André Anjos committed
890

Philip ABBET's avatar
Philip ABBET committed
891
892
893
        if not self._name:
            exc = exc or RuntimeError
            raise exc("algorithm has no name")
894

Samuel GAIST's avatar
Samuel GAIST committed
895
        if self.data["language"] == "unknown":
Philip ABBET's avatar
Philip ABBET committed
896
897
            exc = exc or RuntimeError
            raise exc("algorithm has no programming language set")
898

Philip ABBET's avatar
Philip ABBET committed
899
900
901
902
        if not self.valid:
            message = "cannot load code for invalid algorithm (%s)" % (self.name,)
            exc = exc or RuntimeError
            raise exc(message)
903

Philip ABBET's avatar
Philip ABBET committed
904
905
        # loads the module only once through the lifetime of the algorithm object
        try:
Samuel GAIST's avatar
Samuel GAIST committed
906
907
908
909
910
911
912
913
914
915
            self.__module = getattr(
                self,
                "module",
                loader.load_module(
                    self.name.replace(os.sep, "_"),
                    self.storage.code.path,
                    self.uses_dict(),
                ),
            )
        except Exception:
Philip ABBET's avatar
Philip ABBET committed
916
917
918
919
            if exc is not None:
                type, value, traceback = sys.exc_info()
                six.reraise(exc, exc(value), traceback)
            else:
Samuel GAIST's avatar
Samuel GAIST committed
920
                raise  # just re-raise the user exception
André Anjos's avatar
André Anjos committed
921

Philip ABBET's avatar
Philip ABBET committed
922
        return Runner(self.__module, klass, self, exc)
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938

    def json_dumps(self, indent=4):
        """Dumps the JSON declaration of this object in a string


        Parameters:

          indent (int): The number of indentation spaces at every indentation level


        Returns:

          str: The JSON representation for this object

        """

Samuel GAIST's avatar
Samuel GAIST committed
939
        return simplejson.dumps(self.data, indent=indent, cls=utils.NumpyJSONEncoder)
940
941
942
943
944
945
946
947
948

    def __str__(self):
        return self.json_dumps()

    def write(self, storage=None):
        """Writes contents to prefix location

        Parameters:

949
950
951
          storage (:py:class:`.Storage`, Optional): If you pass a new storage,
            then this object will be written to that storage point rather than
            its default.
952
953
954

        """

Samuel GAIST's avatar
Samuel GAIST committed
955
        if self.data["language"] == "unknown":