algorithm.py 33.5 KB
Newer Older
André Anjos's avatar
André Anjos committed
1
2
3
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

Samuel GAIST's avatar
Samuel GAIST committed
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
###################################################################################
#                                                                                 #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/               #
# Contact: beat.support@idiap.ch                                                  #
#                                                                                 #
# Redistribution and use in source and binary forms, with or without              #
# modification, are permitted provided that the following conditions are met:     #
#                                                                                 #
# 1. Redistributions of source code must retain the above copyright notice, this  #
# list of conditions and the following disclaimer.                                #
#                                                                                 #
# 2. Redistributions in binary form must reproduce the above copyright notice,    #
# this list of conditions and the following disclaimer in the documentation       #
# and/or other materials provided with the distribution.                          #
#                                                                                 #
# 3. Neither the name of the copyright holder nor the names of its contributors   #
# may be used to endorse or promote products derived from this software without   #
# specific prior written permission.                                              #
#                                                                                 #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED   #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE          #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE    #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL      #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR      #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER      #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,   #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE   #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.            #
#                                                                                 #
###################################################################################
André Anjos's avatar
André Anjos committed
35
36


37
38
39
40
41
42
43
"""
=========
algorithm
=========

Validation for algorithms
"""
André Anjos's avatar
André Anjos committed
44
45
46

import os
import sys
47
import logging
André Anjos's avatar
André Anjos committed
48
49
50

import six
import numpy
51
import simplejson as json
André Anjos's avatar
André Anjos committed
52
53
54
55

from . import dataformat
from . import library
from . import loader
56
57
58
from . import utils


59
60
61
logger = logging.getLogger(__name__)


62
# ----------------------------------------------------------
Philip ABBET's avatar
Philip ABBET committed
63

64
65

class Storage(utils.CodeStorage):
Philip ABBET's avatar
Philip ABBET committed
66
    """Resolves paths for algorithms
67

Philip ABBET's avatar
Philip ABBET committed
68
    Parameters:
69

70
      prefix (str): Establishes the prefix of your installation.
71

Philip ABBET's avatar
Philip ABBET committed
72
73
      name (str): The name of the algorithm object in the format
        ``<user>/<name>/<version>``.
74

Philip ABBET's avatar
Philip ABBET committed
75
    """
76

77
78
79
    asset_type = "algorithm"
    asset_folder = "algorithms"

Philip ABBET's avatar
Philip ABBET committed
80
    def __init__(self, prefix, name, language=None):
81

Samuel GAIST's avatar
Samuel GAIST committed
82
        if name.count("/") != 2:
Philip ABBET's avatar
Philip ABBET committed
83
            raise RuntimeError("invalid algorithm name: `%s'" % name)
84

Samuel GAIST's avatar
Samuel GAIST committed
85
        self.username, self.name, self.version = name.split("/")
Philip ABBET's avatar
Philip ABBET committed
86
        self.fullname = name
87
        self.prefix = prefix
88

89
90
91
        path = utils.hashed_or_simple(
            self.prefix, self.asset_folder, name, suffix=".json"
        )
92
        path = path[:-5]
Philip ABBET's avatar
Philip ABBET committed
93
        super(Storage, self).__init__(path, language)
94

André Anjos's avatar
André Anjos committed
95

96
# ----------------------------------------------------------
Philip ABBET's avatar
Philip ABBET committed
97

André Anjos's avatar
André Anjos committed
98
99

class Runner(object):
100
    """A special loader class for algorithms, with specialized methods
André Anjos's avatar
André Anjos committed
101

Philip ABBET's avatar
Philip ABBET committed
102
    Parameters:
André Anjos's avatar
André Anjos committed
103

104
105
      module (:std:term:`module`): The preloaded module containing the
        algorithm as returned by :py:func:`.loader.load_module`.
André Anjos's avatar
André Anjos committed
106

107
108
      obj_name (str): The name of the object within the module you're
        interested on
André Anjos's avatar
André Anjos committed
109

Philip ABBET's avatar
Philip ABBET committed
110
111
112
      algorithm (object): The algorithm instance that is used for parameter
        checking.

113
      exc (:std:term:`class`): The class to use as base exception when
114
        translating the exception from the user code. Read the documentation of
115
        :py:func:`.loader.run` for more details.
André Anjos's avatar
André Anjos committed
116

117
    """
André Anjos's avatar
André Anjos committed
118

119
    def __init__(self, module, obj_name, algorithm, exc=None):
André Anjos's avatar
André Anjos committed
120

Philip ABBET's avatar
Philip ABBET committed
121
122
        try:
            class_ = getattr(module, obj_name)
Samuel GAIST's avatar
Samuel GAIST committed
123
        except Exception:
Philip ABBET's avatar
Philip ABBET committed
124
125
126
127
            if exc is not None:
                type, value, traceback = sys.exc_info()
                six.reraise(exc, exc(value), traceback)
            else:
128
                raise  # Just re-raise the user exception
André Anjos's avatar
André Anjos committed
129

Samuel GAIST's avatar
Samuel GAIST committed
130
        self.obj = loader.run(class_, "__new__", exc)
Philip ABBET's avatar
Philip ABBET committed
131
132
133
        self.name = module.__name__
        self.algorithm = algorithm
        self.exc = exc
André Anjos's avatar
André Anjos committed
134

Samuel GAIST's avatar
Samuel GAIST committed
135
        self.ready = not hasattr(self.obj, "setup")
136
137

        has_api_v1 = self.algorithm.api_version == 1
Samuel GAIST's avatar
Samuel GAIST committed
138
        has_prepare = hasattr(self.obj, "prepare")
139
140
141
142
143
144

        self.prepared = has_api_v1 or not has_prepare

        if has_api_v1 and has_prepare:
            logger.warning("Prepare is a reserved function name starting with API V2")

Philip ABBET's avatar
Philip ABBET committed
145
146
    def _check_parameters(self, parameters):
        """Checks input parameters from the user and fill defaults"""
André Anjos's avatar
André Anjos committed
147

Philip ABBET's avatar
Philip ABBET committed
148
149
150
        user_keys = set(parameters.keys())
        algo_parameters = self.algorithm.parameters or {}
        valid_keys = set(algo_parameters.keys())
André Anjos's avatar
André Anjos committed
151

152
        # Checks the user is not trying to set an undeclared parameter
Philip ABBET's avatar
Philip ABBET committed
153
154
        if not user_keys.issubset(valid_keys):
            err_keys = user_keys - valid_keys
Samuel GAIST's avatar
Samuel GAIST committed
155
156
157
158
159
            message = (
                "parameters `%s' are not declared for algorithm `%s' - "
                "valid parameters are `%s'"
                % (",".join(err_keys), self.name, ",".join(valid_keys))
            )
Philip ABBET's avatar
Philip ABBET committed
160
161
            exc = self.exc or KeyError
            raise exc(message)
André Anjos's avatar
André Anjos committed
162

Philip ABBET's avatar
Philip ABBET committed
163
        retval = dict()
André Anjos's avatar
André Anjos committed
164

Philip ABBET's avatar
Philip ABBET committed
165
        for key, definition in algo_parameters.items():
André Anjos's avatar
André Anjos committed
166

Philip ABBET's avatar
Philip ABBET committed
167
168
169
170
            if key in parameters:
                try:
                    value = self.algorithm.clean_parameter(key, parameters[key])
                except Exception as e:
Samuel GAIST's avatar
Samuel GAIST committed
171
172
173
174
                    message = (
                        "parameter `%s' cannot be safely cast to the declared "
                        "type on algorithm `%s': %s" % (key, self.name, e)
                    )
Philip ABBET's avatar
Philip ABBET committed
175
176
                    exc = self.exc or ValueError
                    raise exc(message)
André Anjos's avatar
André Anjos committed
177

178
            else:  # User has not set a value, use the default
Samuel GAIST's avatar
Samuel GAIST committed
179
                value = algo_parameters[key]["default"]
André Anjos's avatar
André Anjos committed
180

181
            # In the end, set the value on the dictionary to be returned
Philip ABBET's avatar
Philip ABBET committed
182
            retval[key] = value
André Anjos's avatar
André Anjos committed
183

Philip ABBET's avatar
Philip ABBET committed
184
        return retval
André Anjos's avatar
André Anjos committed
185

186
    def setup(self, parameters):
187
        """Sets up the algorithm, only effective on the first call"""
André Anjos's avatar
André Anjos committed
188

189
        # Only effective on the first call
Philip ABBET's avatar
Philip ABBET committed
190
191
        if self.ready:
            return self.ready
André Anjos's avatar
André Anjos committed
192

Philip ABBET's avatar
Philip ABBET committed
193
        completed_parameters = self._check_parameters(parameters)
André Anjos's avatar
André Anjos committed
194

Samuel GAIST's avatar
Samuel GAIST committed
195
        self.ready = loader.run(self.obj, "setup", self.exc, completed_parameters)
Philip ABBET's avatar
Philip ABBET committed
196
197
        return self.ready

198
    def prepare(self, data_loaders):
199
        """Let the algorithm process the data on the non-principal channels"""
Philip ABBET's avatar
Philip ABBET committed
200

201
202
203
204
205
        # Only effective on the first call
        if self.prepared:
            return self.prepared

        # setup() must have run
Philip ABBET's avatar
Philip ABBET committed
206
207
        if not self.ready:
            exc = self.exc or RuntimeError
208
            raise exc("Algorithm '%s' is not yet setup" % self.name)
Philip ABBET's avatar
Philip ABBET committed
209

210
211
212
        # Not available in API version 1
        if self.algorithm.api_version == 1:
            self.prepared = True
Philip ABBET's avatar
Philip ABBET committed
213
            return True
André Anjos's avatar
André Anjos committed
214

215
        # The method is optional
Samuel GAIST's avatar
Samuel GAIST committed
216
        if hasattr(self.obj, "prepare"):
217
            if self.algorithm.type in [Algorithm.AUTONOMOUS, Algorithm.LOOP_USER]:
Samuel GAIST's avatar
Samuel GAIST committed
218
219
220
                self.prepared = loader.run(
                    self.obj, "prepare", self.exc, data_loaders.secondaries()
                )
221
            else:
Samuel GAIST's avatar
Samuel GAIST committed
222
                self.prepared = loader.run(self.obj, "prepare", self.exc, data_loaders)
223
224
        else:
            self.prepared = True
Philip ABBET's avatar
Philip ABBET committed
225

226
        return self.prepared
André Anjos's avatar
André Anjos committed
227

Samuel GAIST's avatar
Samuel GAIST committed
228
229
230
231
232
233
234
235
    def process(
        self,
        inputs=None,
        data_loaders=None,
        outputs=None,
        output=None,
        loop_channel=None,
    ):
236
        """Runs through data"""
André Anjos's avatar
André Anjos committed
237

238
239
240
241
        exc = self.exc or RuntimeError

        def _check_argument(argument, name):
            if argument is None:
Samuel GAIST's avatar
Samuel GAIST committed
242
                raise exc("Missing argument: %s" % name)
243
244

        # setup() must have run
Philip ABBET's avatar
Philip ABBET committed
245
        if not self.ready:
246
            raise exc("Algorithm '%s' is not yet setup" % self.name)
André Anjos's avatar
André Anjos committed
247

248
249
250
251
252
253
        # prepare() must have run
        if not self.prepared:
            raise exc("Algorithm '%s' is not yet prepared" % self.name)

        # Call the correct version of process()
        if self.algorithm.isAnalyzer:
Samuel GAIST's avatar
Samuel GAIST committed
254
            _check_argument(output, "output")
255
256
            outputs_to_use = output
        else:
Samuel GAIST's avatar
Samuel GAIST committed
257
            _check_argument(outputs, "outputs")
258
259
260
            outputs_to_use = outputs

        if self.algorithm.type == Algorithm.LEGACY:
Samuel GAIST's avatar
Samuel GAIST committed
261
262
            _check_argument(inputs, "inputs")
            return loader.run(self.obj, "process", self.exc, inputs, outputs_to_use)
263
264

        else:
Samuel GAIST's avatar
Samuel GAIST committed
265
            _check_argument(data_loaders, "data_loaders")
266
267

            if self.algorithm.type == Algorithm.SEQUENTIAL:
Samuel GAIST's avatar
Samuel GAIST committed
268
                _check_argument(inputs, "inputs")
269

Samuel GAIST's avatar
Samuel GAIST committed
270
271
272
                return loader.run(
                    self.obj, "process", self.exc, inputs, data_loaders, outputs_to_use
                )
273

274
            elif self.algorithm.is_autonomous:
Samuel GAIST's avatar
Samuel GAIST committed
275
                run_args = [self.obj, "process", self.exc, data_loaders, outputs_to_use]
276

Samuel GAIST's avatar
Samuel GAIST committed
277
                has_loop_arg = utils.has_argument(self.obj.process, "loop_channel")
278
                if loop_channel is not None:
279
                    if has_loop_arg:
280
281
                        run_args.append(loop_channel)
                    else:
Samuel GAIST's avatar
Samuel GAIST committed
282
283
284
285
                        raise exc(
                            "Algorithm '%s' is not a valid loop enabled algorithm"
                            % self.name
                        )
286
                elif has_loop_arg:
Samuel GAIST's avatar
Samuel GAIST committed
287
288
289
290
                    raise exc(
                        "Algorithm '%s' is a loop enabled algorithm but no loop_channel given"
                        % self.name
                    )
291
292

                return loader.run(*run_args)
293
294

            else:
Samuel GAIST's avatar
Samuel GAIST committed
295
                raise exc("Unknown algorithm type: %s" % self.algorithm.type)
André Anjos's avatar
André Anjos committed
296

297
298
299
300
301
302
    def validate(self, result):
        """Validates the given results"""

        exc = self.exc or RuntimeError

        if self.algorithm.type != Algorithm.LOOP:
Samuel GAIST's avatar
Samuel GAIST committed
303
            raise exc("Wrong algorithm type: %s" % self.algorithm.type)
304
305
306
307
308
309
310
311
312

        # setup() must have run
        if not self.ready:
            raise exc("Algorithm '%s' is not yet setup" % self.name)

        # prepare() must have run
        if not self.prepared:
            raise exc("Algorithm '%s' is not yet prepared" % self.name)

Samuel GAIST's avatar
Samuel GAIST committed
313
        answer = loader.run(self.obj, "validate", self.exc, result)
314
315
316
317
318

        if not isinstance(answer, tuple):
            raise exc("Improper implementation: validate must return a tuple")

        return answer
319

Philip ABBET's avatar
Philip ABBET committed
320
    def __getattr__(self, key):
321
322
        """Returns an attribute of the algorithm - only called at last resort
        """
Philip ABBET's avatar
Philip ABBET committed
323
        return getattr(self.obj, key)
André Anjos's avatar
André Anjos committed
324
325


326
# ----------------------------------------------------------
Philip ABBET's avatar
Philip ABBET committed
327

328

André Anjos's avatar
André Anjos committed
329
class Algorithm(object):
Philip ABBET's avatar
Philip ABBET committed
330
    """Algorithms represent runnable components within the platform.
André Anjos's avatar
André Anjos committed
331

Philip ABBET's avatar
Philip ABBET committed
332
333
    This class can only parse the meta-parameters of the algorithm (i.e., input
    and output declaration, grouping, synchronization details, parameters and
334
335
    splittability). The actual algorithm is not directly treated by this class.
    It can, however, provide you with a loader for actually running the
Philip ABBET's avatar
Philip ABBET committed
336
    algorithmic code (see :py:meth:`Algorithm.runner`).
André Anjos's avatar
André Anjos committed
337
338


Philip ABBET's avatar
Philip ABBET committed
339
    Parameters:
André Anjos's avatar
André Anjos committed
340

341
      prefix (str): Establishes the prefix of your installation.
André Anjos's avatar
André Anjos committed
342

Philip ABBET's avatar
Philip ABBET committed
343
      name (str): The fully qualified algorithm name (e.g. ``user/algo/1``)
André Anjos's avatar
André Anjos committed
344

345
346
347
348
      dataformat_cache (:py:class:`dict`, Optional): A dictionary mapping
        dataformat names to loaded dataformats. This parameter is optional and,
        if passed, may greatly speed-up algorithm loading times as dataformats
        that are already loaded may be re-used.
André Anjos's avatar
André Anjos committed
349

350
351
352
353
      library_cache (:py:class:`dict`, Optional): A dictionary mapping library
        names to loaded libraries. This parameter is optional and, if passed,
        may greatly speed-up library loading times as libraries that are
        already loaded may be re-used.
André Anjos's avatar
André Anjos committed
354
355


Philip ABBET's avatar
Philip ABBET committed
356
    Attributes:
André Anjos's avatar
André Anjos committed
357

Philip ABBET's avatar
Philip ABBET committed
358
      name (str): The algorithm name
André Anjos's avatar
André Anjos committed
359

360
361
      dataformats (dict): A dictionary containing all pre-loaded dataformats
        used by this algorithm. Data format objects will be of type
362
        :py:class:`.dataformat.DataFormat`.
André Anjos's avatar
André Anjos committed
363

364
365
      libraries (dict): A mapping object defining other libraries this
        algorithm needs to load so it can work properly.
André Anjos's avatar
André Anjos committed
366

Philip ABBET's avatar
Philip ABBET committed
367
368
      uses (dict): A mapping object defining the required library import name
        (keys) and the full-names (values).
André Anjos's avatar
André Anjos committed
369

370
371
      parameters (dict): A dictionary containing all pre-defined parameters
        that this algorithm accepts.
André Anjos's avatar
André Anjos committed
372

Philip ABBET's avatar
Philip ABBET committed
373
374
      splittable (bool): A boolean value that indicates if this algorithm is
        automatically parallelizeable by our backend.
André Anjos's avatar
André Anjos committed
375

Philip ABBET's avatar
Philip ABBET committed
376
      input_map (dict): A dictionary where the key is the input name and the
377
378
        value, its type. All input names (potentially from different groups)
        are comprised in this dictionary.
André Anjos's avatar
André Anjos committed
379

Philip ABBET's avatar
Philip ABBET committed
380
      output_map (dict): A dictionary where the key is the output name and the
381
382
        value, its type. All output names (potentially from different groups)
        are comprised in this dictionary.
André Anjos's avatar
André Anjos committed
383

384
385
386
387
      results (dict): If this algorithm is actually an analyzer (i.e., there
        are no formal outputs, but results that must be saved by the platform),
        then this dictionary contains the names and data types of those
        elements.
André Anjos's avatar
André Anjos committed
388

Philip ABBET's avatar
Philip ABBET committed
389
390
      groups (dict): A list containing dictionaries with inputs and outputs
        belonging to the same synchronization group.
André Anjos's avatar
André Anjos committed
391

Philip ABBET's avatar
Philip ABBET committed
392
393
      errors (list): A list containing errors found while loading this
        algorithm.
394

Philip ABBET's avatar
Philip ABBET committed
395
396
      data (dict): The original data for this algorithm, as loaded by our JSON
        decoder.
André Anjos's avatar
André Anjos committed
397

Philip ABBET's avatar
Philip ABBET committed
398
399
      code (str): The code that is associated with this algorithm, loaded as a
        text (or binary) file.
André Anjos's avatar
André Anjos committed
400

Philip ABBET's avatar
Philip ABBET committed
401
    """
André Anjos's avatar
André Anjos committed
402

Samuel GAIST's avatar
Samuel GAIST committed
403
404
405
406
407
    LEGACY = "legacy"
    SEQUENTIAL = "sequential"
    AUTONOMOUS = "autonomous"
    LOOP = "loop"
    LOOP_USER = "loop_user"
Philip ABBET's avatar
Philip ABBET committed
408

Philip ABBET's avatar
Philip ABBET committed
409
    def __init__(self, prefix, name, dataformat_cache=None, library_cache=None):
André Anjos's avatar
André Anjos committed
410

Philip ABBET's avatar
Philip ABBET committed
411
412
413
414
415
416
417
        self._name = None
        self.storage = None
        self.prefix = prefix
        self.dataformats = {}
        self.libraries = {}
        self.groups = []
        self.errors = []
André Anjos's avatar
André Anjos committed
418

Philip ABBET's avatar
Philip ABBET committed
419
420
        dataformat_cache = dataformat_cache if dataformat_cache is not None else {}
        library_cache = library_cache if library_cache is not None else {}
André Anjos's avatar
André Anjos committed
421

Philip ABBET's avatar
Philip ABBET committed
422
        self._load(name, dataformat_cache, library_cache)
André Anjos's avatar
André Anjos committed
423

Philip ABBET's avatar
Philip ABBET committed
424
425
    def _load(self, data, dataformat_cache, library_cache):
        """Loads the algorithm"""
426

Philip ABBET's avatar
Philip ABBET committed
427
        self._name = data
428

Philip ABBET's avatar
Philip ABBET committed
429
430
431
        self.storage = Storage(self.prefix, data)
        json_path = self.storage.json.path
        if not self.storage.exists():
Samuel GAIST's avatar
Samuel GAIST committed
432
            self.errors.append("Algorithm declaration file not found: %s" % json_path)
Philip ABBET's avatar
Philip ABBET committed
433
            return
434

Samuel GAIST's avatar
Samuel GAIST committed
435
        with open(json_path, "rb") as f:
436
437
438
439
440
441
442
443
            try:
                self.data = json.loads(
                    f.read().decode("utf-8"),
                    object_pairs_hook=utils.error_on_duplicate_key_hook,
                )
            except RuntimeError as error:
                self.errors.append("Algorithm declaration file invalid: %s" % error)
                return
444

Philip ABBET's avatar
Philip ABBET committed
445
        self.code_path = self.storage.code.path
446
        self.code = self.storage.code.load()
André Anjos's avatar
André Anjos committed
447

Samuel GAIST's avatar
Samuel GAIST committed
448
        self.groups = self.data["groups"]
André Anjos's avatar
André Anjos committed
449

Philip ABBET's avatar
Philip ABBET committed
450
        # create maps for easy access to data
Samuel GAIST's avatar
Samuel GAIST committed
451
452
453
454
455
456
457
458
459
460
461
462
463
        self.input_map = dict(
            [(k, v["type"]) for g in self.groups for k, v in g["inputs"].items()]
        )
        self.output_map = dict(
            [
                (k, v["type"])
                for g in self.groups
                for k, v in g.get("outputs", {}).items()
            ]
        )
        self.loop_map = dict(
            [(k, v["type"]) for g in self.groups for k, v in g.get("loop", {}).items()]
        )
André Anjos's avatar
André Anjos committed
464

Philip ABBET's avatar
Philip ABBET committed
465
466
467
        self._load_dataformats(dataformat_cache)
        self._convert_parameter_types()
        self._load_libraries(library_cache)
André Anjos's avatar
André Anjos committed
468

Philip ABBET's avatar
Philip ABBET committed
469
470
471
    def _load_dataformats(self, dataformat_cache):
        """Makes sure we can load all requested formats
        """
André Anjos's avatar
André Anjos committed
472

Philip ABBET's avatar
Philip ABBET committed
473
        for group in self.groups:
André Anjos's avatar
André Anjos committed
474

Samuel GAIST's avatar
Samuel GAIST committed
475
476
477
            for name, input in group["inputs"].items():
                if input["type"] in self.dataformats:
                    continue
André Anjos's avatar
André Anjos committed
478

Samuel GAIST's avatar
Samuel GAIST committed
479
480
                if dataformat_cache and input["type"] in dataformat_cache:  # reuse
                    thisformat = dataformat_cache[input["type"]]
481
                else:  # load it
Samuel GAIST's avatar
Samuel GAIST committed
482
                    thisformat = dataformat.DataFormat(self.prefix, input["type"])
483
                    if dataformat_cache is not None:  # update it
Samuel GAIST's avatar
Samuel GAIST committed
484
                        dataformat_cache[input["type"]] = thisformat
André Anjos's avatar
André Anjos committed
485

Samuel GAIST's avatar
Samuel GAIST committed
486
                self.dataformats[input["type"]] = thisformat
André Anjos's avatar
André Anjos committed
487

Samuel GAIST's avatar
Samuel GAIST committed
488
489
            if "outputs" not in group:
                continue
André Anjos's avatar
André Anjos committed
490

Samuel GAIST's avatar
Samuel GAIST committed
491
492
493
            for name, output in group["outputs"].items():
                if output["type"] in self.dataformats:
                    continue
André Anjos's avatar
André Anjos committed
494

Samuel GAIST's avatar
Samuel GAIST committed
495
496
                if dataformat_cache and output["type"] in dataformat_cache:  # reuse
                    thisformat = dataformat_cache[output["type"]]
497
                else:  # load it
Samuel GAIST's avatar
Samuel GAIST committed
498
499
500
                    thisformat = dataformat.DataFormat(self.prefix, output["type"])
                    if dataformat_cache is not None:  # update it
                        dataformat_cache[output["type"]] = thisformat
André Anjos's avatar
André Anjos committed
501

Samuel GAIST's avatar
Samuel GAIST committed
502
                self.dataformats[output["type"]] = thisformat
André Anjos's avatar
André Anjos committed
503

Samuel GAIST's avatar
Samuel GAIST committed
504
            if "loop" not in group:
505
506
                continue

Samuel GAIST's avatar
Samuel GAIST committed
507
508
            for name, entry in group["loop"].items():
                entry_format = entry["type"]
509
510
511
512
513
514
515
516
517
518
519
520
                if entry_format in self.dataformats:
                    continue

                if dataformat_cache and entry_format in dataformat_cache:
                    thisformat = dataformat_cache[entry_format]
                else:
                    thisformat = dataformat.DataFormat(self.prefix, entry_format)
                    if dataformat_cache is not None:
                        dataformat_cache[entry_format] = thisformat

                self.dataformats[entry_format] = thisformat

Philip ABBET's avatar
Philip ABBET committed
521
        if self.results:
André Anjos's avatar
André Anjos committed
522

Philip ABBET's avatar
Philip ABBET committed
523
            for name, result in self.results.items():
André Anjos's avatar
André Anjos committed
524

Samuel GAIST's avatar
Samuel GAIST committed
525
                if result["type"].find("/") != -1:
André Anjos's avatar
André Anjos committed
526

Samuel GAIST's avatar
Samuel GAIST committed
527
528
                    if result["type"] in self.dataformats:
                        continue
André Anjos's avatar
André Anjos committed
529

Samuel GAIST's avatar
Samuel GAIST committed
530
531
                    if dataformat_cache and result["type"] in dataformat_cache:  # reuse
                        thisformat = dataformat_cache[result["type"]]
Philip ABBET's avatar
Philip ABBET committed
532
                    else:
Samuel GAIST's avatar
Samuel GAIST committed
533
534
535
                        thisformat = dataformat.DataFormat(self.prefix, result["type"])
                        if dataformat_cache is not None:  # update it
                            dataformat_cache[result["type"]] = thisformat
André Anjos's avatar
André Anjos committed
536

Samuel GAIST's avatar
Samuel GAIST committed
537
                    self.dataformats[result["type"]] = thisformat
André Anjos's avatar
André Anjos committed
538

Philip ABBET's avatar
Philip ABBET committed
539
540
541
    def _convert_parameter_types(self):
        """Converts types to numpy equivalents, checks defaults, ranges and choices
        """
André Anjos's avatar
André Anjos committed
542

Philip ABBET's avatar
Philip ABBET committed
543
544
545
546
        def _try_convert(name, tp, value, desc):
            try:
                return tp.type(value)
            except Exception as e:
Samuel GAIST's avatar
Samuel GAIST committed
547
548
549
550
                self.errors.append(
                    "%s for parameter `%s' cannot be cast to type "
                    "`%s': %s" % (desc, name, tp.name, e)
                )
André Anjos's avatar
André Anjos committed
551

Samuel GAIST's avatar
Samuel GAIST committed
552
553
        if self.parameters is None:
            return
André Anjos's avatar
André Anjos committed
554

Philip ABBET's avatar
Philip ABBET committed
555
        for name, parameter in self.parameters.items():
Samuel GAIST's avatar
Samuel GAIST committed
556
557
            if parameter["type"] == "string":
                parameter["type"] = numpy.dtype("str")
Philip ABBET's avatar
Philip ABBET committed
558
            else:
Samuel GAIST's avatar
Samuel GAIST committed
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
                parameter["type"] = numpy.dtype(parameter["type"])

            if "range" in parameter:
                parameter["range"][0] = _try_convert(
                    name, parameter["type"], parameter["range"][0], "start of range"
                )
                parameter["range"][1] = _try_convert(
                    name, parameter["type"], parameter["range"][1], "end of range"
                )
                if parameter["range"][0] >= parameter["range"][1]:
                    self.errors.append(
                        "range for parameter `%s' has a start greater "
                        "then the end value (%r >= %r)"
                        % (name, parameter["range"][0], parameter["range"][1])
                    )

            if "choice" in parameter:
                for i, choice in enumerate(parameter["choice"]):
                    parameter["choice"][i] = _try_convert(
                        name,
                        parameter["type"],
                        parameter["choice"][i],
                        "choice[%d]" % i,
                    )

            if "default" in parameter:
                parameter["default"] = _try_convert(
                    name, parameter["type"], parameter["default"], "default"
                )

                if "range" in parameter:  # check range
                    if (
                        parameter["default"] < parameter["range"][0]
                        or parameter["default"] > parameter["range"][1]
                    ):
                        self.errors.append(
                            "default for parameter `%s' (%r) is not "
                            "within parameter range [%r, %r]"
                            % (
                                name,
                                parameter["default"],
                                parameter["range"][0],
                                parameter["range"][1],
                            )
                        )
André Anjos's avatar
André Anjos committed
604

Samuel GAIST's avatar
Samuel GAIST committed
605
606
607
608
609
610
611
612
613
614
615
                if "choice" in parameter:  # check choices
                    if parameter["default"] not in parameter["choice"]:
                        self.errors.append(
                            "default for parameter `%s' (%r) is not "
                            "a valid choice `[%s]'"
                            % (
                                name,
                                parameter["default"],
                                ", ".join(["%r" % k for k in parameter["choice"]]),
                            )
                        )
André Anjos's avatar
André Anjos committed
616

Philip ABBET's avatar
Philip ABBET committed
617
    def _load_libraries(self, library_cache):
André Anjos's avatar
André Anjos committed
618

Philip ABBET's avatar
Philip ABBET committed
619
        # all used libraries must be loadable; cannot use self as a library
André Anjos's avatar
André Anjos committed
620

Philip ABBET's avatar
Philip ABBET committed
621
        if self.uses:
André Anjos's avatar
André Anjos committed
622

Philip ABBET's avatar
Philip ABBET committed
623
            for name, value in self.uses.items():
André Anjos's avatar
André Anjos committed
624

Samuel GAIST's avatar
Samuel GAIST committed
625
626
627
                self.libraries[value] = library_cache.setdefault(
                    value, library.Library(self.prefix, value, library_cache)
                )
André Anjos's avatar
André Anjos committed
628

Philip ABBET's avatar
Philip ABBET committed
629
630
631
632
    @property
    def name(self):
        """Returns the name of this object
        """
Samuel GAIST's avatar
Samuel GAIST committed
633
        return self._name or "__unnamed_algorithm__"
634

Philip ABBET's avatar
Philip ABBET committed
635
636
    @name.setter
    def name(self, value):
637
638
639
640
641
        """Sets the name of this object

        Parameters:
            name (str): Name of this object
        """
642

Samuel GAIST's avatar
Samuel GAIST committed
643
        if self.data["language"] == "unknown":
Philip ABBET's avatar
Philip ABBET committed
644
            raise RuntimeError("algorithm has no programming language set")
645

Philip ABBET's avatar
Philip ABBET committed
646
        self._name = value
Samuel GAIST's avatar
Samuel GAIST committed
647
        self.storage = Storage(self.prefix, value, self.data["language"])
André Anjos's avatar
André Anjos committed
648

Philip ABBET's avatar
Philip ABBET committed
649
650
651
    @property
    def schema_version(self):
        """Returns the schema version"""
Samuel GAIST's avatar
Samuel GAIST committed
652
        return self.data.get("schema_version", 1)
André Anjos's avatar
André Anjos committed
653

Philip ABBET's avatar
Philip ABBET committed
654
655
656
    @property
    def language(self):
        """Returns the current language set for the executable code"""
Samuel GAIST's avatar
Samuel GAIST committed
657
        return self.data["language"]
658

Philip ABBET's avatar
Philip ABBET committed
659
660
661
    @property
    def api_version(self):
        """Returns the API version"""
Samuel GAIST's avatar
Samuel GAIST committed
662
        return self.data.get("api_version", 1)
Philip ABBET's avatar
Philip ABBET committed
663
664
665
666
667
668
669

    @property
    def type(self):
        """Returns the type of algorithm"""
        if self.api_version == 1:
            return Algorithm.LEGACY

Samuel GAIST's avatar
Samuel GAIST committed
670
        return self.data.get("type", Algorithm.SEQUENTIAL)
Philip ABBET's avatar
Philip ABBET committed
671

672
673
674
    @property
    def is_autonomous(self):
        """ Returns whether the algorithm is in the autonomous category"""
Samuel GAIST's avatar
Samuel GAIST committed
675
        return self.type in [Algorithm.AUTONOMOUS, Algorithm.LOOP_USER, Algorithm.LOOP]
676

Philip ABBET's avatar
Philip ABBET committed
677
678
679
680
681
    @language.setter
    def language(self, value):
        """Sets the current executable code programming language"""
        if self.storage:
            self.storage.language = value
Samuel GAIST's avatar
Samuel GAIST committed
682
        self.data["language"] = value
683

Philip ABBET's avatar
Philip ABBET committed
684
685
    def clean_parameter(self, parameter, value):
        """Checks if a given value against a declared parameter
André Anjos's avatar
André Anjos committed
686

Philip ABBET's avatar
Philip ABBET committed
687
        This method checks if the provided user value can be safe-cast to the
688
689
        parameter type as defined on its specification and that it conforms to
        any parameter-imposed restrictions.
André Anjos's avatar
André Anjos committed
690
691


Philip ABBET's avatar
Philip ABBET committed
692
        Parameters:
André Anjos's avatar
André Anjos committed
693

Philip ABBET's avatar
Philip ABBET committed
694
          parameter (str): The name of the parameter to check the value against
André Anjos's avatar
André Anjos committed
695

Philip ABBET's avatar
Philip ABBET committed
696
697
          value (object): An object that will be safe cast into the defined
            parameter type.
André Anjos's avatar
André Anjos committed
698
699


Philip ABBET's avatar
Philip ABBET committed
700
        Returns:
André Anjos's avatar
André Anjos committed
701

Philip ABBET's avatar
Philip ABBET committed
702
          The converted value, with an appropriate numpy type.
André Anjos's avatar
André Anjos committed
703
704


Philip ABBET's avatar
Philip ABBET committed
705
        Raises:
André Anjos's avatar
André Anjos committed
706

Philip ABBET's avatar
Philip ABBET committed
707
708
          KeyError: If the parameter cannot be found on this algorithm's
            declaration.
André Anjos's avatar
André Anjos committed
709

Philip ABBET's avatar
Philip ABBET committed
710
          ValueError: If the parameter cannot be safe cast into the algorithm's
711
712
            type. Alternatively, a ``ValueError`` may also be raised if a range
            or choice was specified and the value does not obey those settings
Philip ABBET's avatar
Philip ABBET committed
713
714
            stipulated for the parameter
        """
André Anjos's avatar
André Anjos committed
715

Philip ABBET's avatar
Philip ABBET committed
716
717
        if (self.parameters is None) or (parameter not in self.parameters):
            raise KeyError(parameter)
André Anjos's avatar
André Anjos committed
718

Samuel GAIST's avatar
Samuel GAIST committed
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
        retval = self.parameters[parameter]["type"].type(value)

        if (
            "choice" in self.parameters[parameter]
            and retval not in self.parameters[parameter]["choice"]
        ):
            raise ValueError(
                "value for `%s' (%r) must be one of `[%s]'"
                % (
                    parameter,
                    value,
                    ", ".join(["%r" % k for k in self.parameters[parameter]["choice"]]),
                )
            )

        if "range" in self.parameters[parameter] and (
            retval < self.parameters[parameter]["range"][0]
            or retval > self.parameters[parameter]["range"][1]
        ):
            raise ValueError(
                "value for `%s' (%r) must be picked within "
                "interval `[%r, %r]'"
                % (
                    parameter,
                    value,
                    self.parameters[parameter]["range"][0],
                    self.parameters[parameter]["range"][1],
                )
            )
André Anjos's avatar
André Anjos committed
748

Philip ABBET's avatar
Philip ABBET committed
749
        return retval
André Anjos's avatar
André Anjos committed
750

Philip ABBET's avatar
Philip ABBET committed
751
752
753
    @property
    def valid(self):
        """A boolean that indicates if this algorithm is valid or not"""
754

Philip ABBET's avatar
Philip ABBET committed
755
        return not bool(self.errors)
756

Philip ABBET's avatar
Philip ABBET committed
757
758
    @property
    def uses(self):
Samuel GAIST's avatar
Samuel GAIST committed
759
        return self.data.get("uses")
André Anjos's avatar
André Anjos committed
760

Philip ABBET's avatar
Philip ABBET committed
761
762
    @uses.setter
    def uses(self, value):
Samuel GAIST's avatar
Samuel GAIST committed
763
764
        if not isinstance(value, dict):
            raise RuntimeError("Invalid uses entry, must be a dict")
Samuel GAIST's avatar
Samuel GAIST committed
765
        self.data["uses"] = value
Philip ABBET's avatar
Philip ABBET committed
766
        return value
767

768
769
    @property
    def isAnalyzer(self):
770
771
        """Returns whether this algorithms is an analyzer"""

Samuel GAIST's avatar
Samuel GAIST committed
772
        return self.results is not None
773

Philip ABBET's avatar
Philip ABBET committed
774
775
    @property
    def results(self):
776
777
        """The results of this algorithm"""

Samuel GAIST's avatar
Samuel GAIST committed
778
        return self.data.get("results")
André Anjos's avatar
André Anjos committed
779

Philip ABBET's avatar
Philip ABBET committed
780
781
    @results.setter
    def results(self, value):
Samuel GAIST's avatar
Samuel GAIST committed
782
        self.data["results"] = value
Philip ABBET's avatar
Philip ABBET committed
783
        return value
784

Philip ABBET's avatar
Philip ABBET committed
785
786
    @property
    def parameters(self):
787
788
        """The parameters of this algorithm"""

Samuel GAIST's avatar
Samuel GAIST committed
789
        return self.data.get("parameters")
André Anjos's avatar
André Anjos committed
790

Philip ABBET's avatar
Philip ABBET committed
791
792
    @parameters.setter
    def parameters(self, value):
Samuel GAIST's avatar
Samuel GAIST committed
793
        self.data["parameters"] = value
Philip ABBET's avatar
Philip ABBET committed
794
        return value
795

Philip ABBET's avatar
Philip ABBET committed
796
797
    @property
    def splittable(self):
798
        """Whether this algorithm can be split between several processes"""
Samuel GAIST's avatar
Samuel GAIST committed
799
        return self.data.get("splittable", False)
André Anjos's avatar
André Anjos committed
800

Philip ABBET's avatar
Philip ABBET committed
801
802
    @splittable.setter
    def splittable(self, value):
Samuel GAIST's avatar
Samuel GAIST committed
803
        self.data["splittable"] = value
Philip ABBET's avatar
Philip ABBET committed
804
        return value
805

806
807
808
    @property
    def description(self):
        """The short description for this object"""
Samuel GAIST's avatar
Samuel GAIST committed
809
        return self.data.get("description", None)
810
811
812
813

    @description.setter
    def description(self, value):
        """Sets the short description for this object"""
Samuel GAIST's avatar
Samuel GAIST committed
814
        self.data["description"] = value
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833

    @property
    def documentation(self):
        """The full-length description for this object"""

        if not self._name:
            raise RuntimeError("algorithm has no name")

        if self.storage.doc.exists():
            return self.storage.doc.load()
        return None

    @documentation.setter
    def documentation(self, value):
        """Sets the full-length description for this object"""

        if not self._name:
            raise RuntimeError("algorithm has no name")

Samuel GAIST's avatar
Samuel GAIST committed
834
        if hasattr(value, "read"):
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
            self.storage.doc.save(value.read())
        else:
            self.storage.doc.save(value)

    def hash(self):
        """Returns the hexadecimal hash for the current algorithm"""

        if not self._name:
            raise RuntimeError("algorithm has no name")

        return self.storage.hash()

    def result_dataformat(self):
        """Generates, on-the-fly, the dataformat for the result readout"""

        if not self.results:
Samuel GAIST's avatar
Samuel GAIST committed
851
852
853
            raise TypeError(
                "algorithm `%s' is a block algorithm, not an analyzer" % (self.name)
            )
854

Samuel GAIST's avatar
Samuel GAIST committed
855
856
857
        format = dataformat.DataFormat(
            self.prefix, dict([(k, v["type"]) for k, v in self.results.items()])
        )
858

Samuel GAIST's avatar
Samuel GAIST committed
859
        format.name = "analysis:" + self.name
860
861
862

        return format

Philip ABBET's avatar
Philip ABBET committed
863
864
    def uses_dict(self):
        """Returns the usage dictionary for all dependent modules"""
André Anjos's avatar
André Anjos committed
865

Samuel GAIST's avatar
Samuel GAIST committed
866
        if self.data["language"] == "unknown":
Philip ABBET's avatar
Philip ABBET committed
867
            raise RuntimeError("algorithm has no programming language set")
868

Philip ABBET's avatar
Philip ABBET committed
869
870
        if not self._name:
            raise RuntimeError("algorithm has no name")
871

Philip ABBET's avatar
Philip ABBET committed
872
        retval = {}
André Anjos's avatar
André Anjos committed
873

Philip ABBET's avatar
Philip ABBET committed
874
875
876
        if self.uses is not None:
            for name, value in self.uses.items():
                retval[name] = dict(
Samuel GAIST's avatar
Samuel GAIST committed
877
878
879
                    path=self.libraries[value].storage.code.path,
                    uses=self.libraries[value].uses_dict(),
                )
André Anjos's avatar
André Anjos committed
880

Philip ABBET's avatar
Philip ABBET committed
881
        return retval
André Anjos's avatar
André Anjos committed
882

Samuel GAIST's avatar
Samuel GAIST committed
883
    def runner(self, klass="Algorithm", exc=None):
Philip ABBET's avatar
Philip ABBET committed
884
        """Returns a runnable algorithm object.
André Anjos's avatar
André Anjos committed
885

Philip ABBET's avatar
Philip ABBET committed
886
        Parameters:
André Anjos's avatar
André Anjos committed
887

Philip ABBET's avatar
Philip ABBET committed
888
          klass (str): The name of the class to load the runnable algorithm from
André Anjos's avatar
André Anjos committed
889

890
891
          exc (:std:term:`class`): If passed, must be a valid exception class
            that will be used to report errors in the read-out of this algorithm's code.
André Anjos's avatar
André Anjos committed
892

Philip ABBET's avatar
Philip ABBET committed
893
        Returns:
André Anjos's avatar
André Anjos committed
894

895
          :py:class:`Runner`: An instance of the algorithm,
Philip ABBET's avatar
Philip ABBET committed
896
897
898
            which will be constructed, but not setup.  You **must** set it up
            before using the ``process`` method.
        """
André Anjos's avatar
André Anjos committed
899

Philip ABBET's avatar
Philip ABBET committed
900
901
902
        if not self._name:
            exc = exc or RuntimeError
            raise exc("algorithm has no name")
903

Samuel GAIST's avatar
Samuel GAIST committed
904
        if self.data["language"] == "unknown":
Philip ABBET's avatar
Philip ABBET committed
905
906
            exc = exc or RuntimeError
            raise exc("algorithm has no programming language set")
907

Philip ABBET's avatar
Philip ABBET committed
908
909
910
911
        if not self.valid:
            message = "cannot load code for invalid algorithm (%s)" % (self.name,)
            exc = exc or RuntimeError
            raise exc(message)
912

Philip ABBET's avatar
Philip ABBET committed
913
914
        # loads the module only once through the lifetime of the algorithm object
        try:
Samuel GAIST's avatar
Samuel GAIST committed
915
916
917
918
919
920
921
922
923
924
            self.__module = getattr(
                self,
                "module",
                loader.load_module(
                    self.name.replace(os.sep, "_"),
                    self.storage.code.path,
                    self.uses_dict(),
                ),
            )
        except Exception:
Philip ABBET's avatar
Philip ABBET committed
925
926
927
928
            if exc is not None:
                type, value, traceback = sys.exc_info()
                six.reraise(exc, exc(value), traceback)
            else:
Samuel GAIST's avatar
Samuel GAIST committed
929
                raise  # just re-raise the user exception
André Anjos's avatar
André Anjos committed
930

Philip ABBET's avatar
Philip ABBET committed
931
        return Runner(self.__module, klass, self, exc)
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947

    def json_dumps(self, indent=4):
        """Dumps the JSON declaration of this object in a string


        Parameters:

          indent (int): The number of indentation spaces at every indentation level


        Returns:

          str: The JSON representation for this object

        """

948
        return json.dumps(self.data, indent=indent, cls=utils.NumpyJSONEncoder)
949
950
951
952
953
954
955
956
957