library.py 11.8 KB
Newer Older
André Anjos's avatar
André Anjos committed
1
2
3
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

Samuel GAIST's avatar
Samuel GAIST committed
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
###################################################################################
#                                                                                 #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/               #
# Contact: beat.support@idiap.ch                                                  #
#                                                                                 #
# Redistribution and use in source and binary forms, with or without              #
# modification, are permitted provided that the following conditions are met:     #
#                                                                                 #
# 1. Redistributions of source code must retain the above copyright notice, this  #
# list of conditions and the following disclaimer.                                #
#                                                                                 #
# 2. Redistributions in binary form must reproduce the above copyright notice,    #
# this list of conditions and the following disclaimer in the documentation       #
# and/or other materials provided with the distribution.                          #
#                                                                                 #
# 3. Neither the name of the copyright holder nor the names of its contributors   #
# may be used to endorse or promote products derived from this software without   #
# specific prior written permission.                                              #
#                                                                                 #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED   #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE          #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE    #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL      #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR      #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER      #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,   #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE   #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.            #
#                                                                                 #
###################################################################################
André Anjos's avatar
André Anjos committed
35
36


37
38
39
40
41
42
43
"""
=======
library
=======

Validation for libraries
"""
André Anjos's avatar
André Anjos committed
44
45
46
47
48

import os
import simplejson

from . import loader
49
50
51
from . import utils


52
# ----------------------------------------------------------
53

54
55

class Storage(utils.CodeStorage):
Philip ABBET's avatar
Philip ABBET committed
56
    """Resolves paths for libraries
57

Philip ABBET's avatar
Philip ABBET committed
58
    Parameters:
59

Samuel GAIST's avatar
Samuel GAIST committed
60
      prefix (str): Establishes the prefix of
61
        your installation.
62

Philip ABBET's avatar
Philip ABBET committed
63
64
      name (str): The name of the library object in the format
        ``<user>/<name>/<version>``.
65

Philip ABBET's avatar
Philip ABBET committed
66
    """
67

68
69
70
    asset_type = "library"
    asset_folder = "libraries"

Philip ABBET's avatar
Philip ABBET committed
71
    def __init__(self, prefix, name, language=None):
72

Samuel GAIST's avatar
Samuel GAIST committed
73
        if name.count("/") != 2:
Philip ABBET's avatar
Philip ABBET committed
74
            raise RuntimeError("invalid library name: `%s'" % name)
75

Samuel GAIST's avatar
Samuel GAIST committed
76
        self.username, self.name, self.version = name.split("/")
Philip ABBET's avatar
Philip ABBET committed
77
        self.fullname = name
Samuel GAIST's avatar
Samuel GAIST committed
78
        self.prefix = prefix
79

80
81
82
        path = utils.hashed_or_simple(
            self.prefix, self.asset_folder, name, suffix=".json"
        )
83
        path = path[:-5]
Philip ABBET's avatar
Philip ABBET committed
84
        super(Storage, self).__init__(path, language)
85

André Anjos's avatar
André Anjos committed
86

87
# ----------------------------------------------------------
88

André Anjos's avatar
André Anjos committed
89
90

class Library(object):
Philip ABBET's avatar
Philip ABBET committed
91
    """Librarys represent independent algorithm components within the platform.
André Anjos's avatar
André Anjos committed
92

Philip ABBET's avatar
Philip ABBET committed
93
94
95
    This class can only parse the meta-parameters of the library. The actual
    library is not directly treated by this class - only by the associated
    algorithms.
André Anjos's avatar
André Anjos committed
96
97


Philip ABBET's avatar
Philip ABBET committed
98
    Parameters:
André Anjos's avatar
André Anjos committed
99

Samuel GAIST's avatar
Samuel GAIST committed
100
      prefix (str): Establishes the prefix of your installation.
André Anjos's avatar
André Anjos committed
101

Philip ABBET's avatar
Philip ABBET committed
102
      name (str): The fully qualified algorithm name (e.g. ``user/algo/1``)
André Anjos's avatar
André Anjos committed
103

104
105
106
107
      library_cache (:py:class:`dict`, Optional): A dictionary mapping library
        names to loaded libraries. This parameter is optional and, if passed,
        may greatly speed-up library loading times as libraries that are
        already loaded may be re-used.
André Anjos's avatar
André Anjos committed
108
109


Philip ABBET's avatar
Philip ABBET committed
110
    Attributes:
André Anjos's avatar
André Anjos committed
111

Philip ABBET's avatar
Philip ABBET committed
112
      name (str): The library name
André Anjos's avatar
André Anjos committed
113

Philip ABBET's avatar
Philip ABBET committed
114
115
      description (str): The short description string, loaded from the JSON
        file if one was set.
116

Philip ABBET's avatar
Philip ABBET committed
117
      documentation (str): The full-length docstring for this object.
118

Philip ABBET's avatar
Philip ABBET committed
119
120
      storage (object): A simple object that provides information about file
        paths for this library
121

Philip ABBET's avatar
Philip ABBET committed
122
123
      libraries (dict): A mapping object defining other libraries this library
        needs to load so it can work properly.
André Anjos's avatar
André Anjos committed
124

Philip ABBET's avatar
Philip ABBET committed
125
126
      uses (dict): A mapping object defining the required library import name
        (keys) and the full-names (values).
André Anjos's avatar
André Anjos committed
127

Philip ABBET's avatar
Philip ABBET committed
128
129
      errors (list): A list containing errors found while loading this
        library.
130

Philip ABBET's avatar
Philip ABBET committed
131
132
      data (dict): The original data for this library, as loaded by our JSON
        decoder.
André Anjos's avatar
André Anjos committed
133

Philip ABBET's avatar
Philip ABBET committed
134
135
      code (str): The code that is associated with this library, loaded as a
        text (or binary) file.
André Anjos's avatar
André Anjos committed
136

Philip ABBET's avatar
Philip ABBET committed
137
    """
André Anjos's avatar
André Anjos committed
138

Philip ABBET's avatar
Philip ABBET committed
139
    def __init__(self, prefix, name, library_cache=None):
André Anjos's avatar
André Anjos committed
140

Philip ABBET's avatar
Philip ABBET committed
141
142
143
144
145
        self._name = None
        self.storage = None
        self.prefix = prefix
        self.errors = []
        self.libraries = {}
André Anjos's avatar
André Anjos committed
146

Philip ABBET's avatar
Philip ABBET committed
147
        library_cache = library_cache if library_cache is not None else {}
André Anjos's avatar
André Anjos committed
148

Philip ABBET's avatar
Philip ABBET committed
149
150
151
        try:
            self._load(name, library_cache)
        finally:
Samuel GAIST's avatar
Samuel GAIST committed
152
            if self._name is not None:  # registers it into the cache, even if failed
Philip ABBET's avatar
Philip ABBET committed
153
                library_cache[self._name] = self
154

Philip ABBET's avatar
Philip ABBET committed
155
156
    def _load(self, data, library_cache):
        """Loads the library"""
157

Philip ABBET's avatar
Philip ABBET committed
158
        self._name = data
159

Philip ABBET's avatar
Philip ABBET committed
160
161
162
        self.storage = Storage(self.prefix, data)
        json_path = self.storage.json.path
        if not self.storage.exists():
Samuel GAIST's avatar
Samuel GAIST committed
163
            self.errors.append("Library declaration file not found: %s" % json_path)
Philip ABBET's avatar
Philip ABBET committed
164
            return
165

Samuel GAIST's avatar
Samuel GAIST committed
166
167
        with open(json_path, "rb") as f:
            self.data = simplejson.loads(f.read().decode("utf-8"))
André Anjos's avatar
André Anjos committed
168

Philip ABBET's avatar
Philip ABBET committed
169
        self.code_path = self.storage.code.path
André Anjos's avatar
André Anjos committed
170

Philip ABBET's avatar
Philip ABBET committed
171
        # if no errors so far, make sense out of the library data
Samuel GAIST's avatar
Samuel GAIST committed
172
        self.data.setdefault("uses", {})
André Anjos's avatar
André Anjos committed
173

Philip ABBET's avatar
Philip ABBET committed
174
175
176
        if self.uses is not None:
            for name, value in self.uses.items():
                self.libraries[value] = Library(self.prefix, value, library_cache)
André Anjos's avatar
André Anjos committed
177

Philip ABBET's avatar
Philip ABBET committed
178
        self.libraries[self._name] = self
André Anjos's avatar
André Anjos committed
179

Philip ABBET's avatar
Philip ABBET committed
180
181
    def uses_dict(self):
        """Returns the usage dictionary for all dependent modules"""
André Anjos's avatar
André Anjos committed
182

Samuel GAIST's avatar
Samuel GAIST committed
183
        if self.data["language"] == "unknown":
Philip ABBET's avatar
Philip ABBET committed
184
            raise RuntimeError("library has no programming language set")
185

Philip ABBET's avatar
Philip ABBET committed
186
187
        if not self._name:
            raise RuntimeError("library has no name")
188

Philip ABBET's avatar
Philip ABBET committed
189
        retval = {}
André Anjos's avatar
André Anjos committed
190

Philip ABBET's avatar
Philip ABBET committed
191
        if self.uses is not None:
André Anjos's avatar
André Anjos committed
192

Philip ABBET's avatar
Philip ABBET committed
193
194
            for name, value in self.uses.items():
                retval[name] = dict(
Samuel GAIST's avatar
Samuel GAIST committed
195
196
197
                    path=self.libraries[value].storage.code.path,
                    uses=self.libraries[value].uses_dict(),
                )
André Anjos's avatar
André Anjos committed
198

Philip ABBET's avatar
Philip ABBET committed
199
        return retval
André Anjos's avatar
André Anjos committed
200

Philip ABBET's avatar
Philip ABBET committed
201
202
    def load(self):
        """Loads the Python module for this library resolving all references
André Anjos's avatar
André Anjos committed
203

Philip ABBET's avatar
Philip ABBET committed
204
205
        Returns the loaded Python module.
        """
André Anjos's avatar
André Anjos committed
206

Samuel GAIST's avatar
Samuel GAIST committed
207
        if self.data["language"] == "unknown":
Philip ABBET's avatar
Philip ABBET committed
208
            raise RuntimeError("library has no programming language set")
209

Philip ABBET's avatar
Philip ABBET committed
210
211
        if not self._name:
            raise RuntimeError("library has no name")
212

Samuel GAIST's avatar
Samuel GAIST committed
213
214
215
        return loader.load_module(
            self.name.replace(os.sep, "_"), self.storage.code.path, self.uses_dict()
        )
216

Philip ABBET's avatar
Philip ABBET committed
217
218
    @property
    def name(self):
219
220
        """Returns the name of this object"""

Samuel GAIST's avatar
Samuel GAIST committed
221
        return self._name or "__unnamed_library__"
222

Philip ABBET's avatar
Philip ABBET committed
223
224
    @name.setter
    def name(self, value):
225

Samuel GAIST's avatar
Samuel GAIST committed
226
        if self.data["language"] == "unknown":
Philip ABBET's avatar
Philip ABBET committed
227
            raise RuntimeError("library has no programming language set")
228

Philip ABBET's avatar
Philip ABBET committed
229
        self._name = value
Samuel GAIST's avatar
Samuel GAIST committed
230
        self.storage = Storage(self.prefix, value, self.data["language"])
André Anjos's avatar
André Anjos committed
231

Philip ABBET's avatar
Philip ABBET committed
232
233
234
    @property
    def schema_version(self):
        """Returns the schema version"""
Samuel GAIST's avatar
Samuel GAIST committed
235
        return self.data.get("schema_version", 1)
André Anjos's avatar
André Anjos committed
236

Philip ABBET's avatar
Philip ABBET committed
237
238
239
    @property
    def language(self):
        """Returns the current language set for the library code"""
Samuel GAIST's avatar
Samuel GAIST committed
240
        return self.data["language"]
241

Philip ABBET's avatar
Philip ABBET committed
242
243
244
    @language.setter
    def language(self, value):
        """Sets the current executable code programming language"""
Samuel GAIST's avatar
Samuel GAIST committed
245
246
247
        if self.storage:
            self.storage.language = value
        self.data["language"] = value
Philip ABBET's avatar
Philip ABBET committed
248
        self._check_language_consistence()
249

Philip ABBET's avatar
Philip ABBET committed
250
251
252
    @property
    def valid(self):
        """A boolean that indicates if this library is valid or not"""
253

Philip ABBET's avatar
Philip ABBET committed
254
        return not bool(self.errors)
255

Philip ABBET's avatar
Philip ABBET committed
256
257
    @property
    def uses(self):
Samuel GAIST's avatar
Samuel GAIST committed
258
        return self.data.get("uses")
259

Philip ABBET's avatar
Philip ABBET committed
260
261
    @uses.setter
    def uses(self, value):
Samuel GAIST's avatar
Samuel GAIST committed
262
        self.data["uses"] = value
Philip ABBET's avatar
Philip ABBET committed
263
        return value
264

Philip ABBET's avatar
Philip ABBET committed
265
266
267
    @property
    def description(self):
        """The short description for this object"""
Samuel GAIST's avatar
Samuel GAIST committed
268
        return self.data.get("description", None)
269

Philip ABBET's avatar
Philip ABBET committed
270
271
272
    @description.setter
    def description(self, value):
        """Sets the short description for this object"""
Samuel GAIST's avatar
Samuel GAIST committed
273
        self.data["description"] = value
274

Philip ABBET's avatar
Philip ABBET committed
275
276
277
    @property
    def documentation(self):
        """The full-length description for this object"""
278

Philip ABBET's avatar
Philip ABBET committed
279
280
        if not self._name:
            raise RuntimeError("library has no name")
281

Philip ABBET's avatar
Philip ABBET committed
282
283
284
        if self.storage.doc.exists():
            return self.storage.doc.load()
        return None
285

Philip ABBET's avatar
Philip ABBET committed
286
287
288
    @documentation.setter
    def documentation(self, value):
        """Sets the full-length description for this object"""
289

Philip ABBET's avatar
Philip ABBET committed
290
291
        if not self._name:
            raise RuntimeError("library has no name")
292

Samuel GAIST's avatar
Samuel GAIST committed
293
        if hasattr(value, "read"):
Philip ABBET's avatar
Philip ABBET committed
294
295
296
            self.storage.doc.save(value.read())
        else:
            self.storage.doc.save(value)
297

Philip ABBET's avatar
Philip ABBET committed
298
299
    def hash(self):
        """Returns the hexadecimal hash for the current library"""
300

Philip ABBET's avatar
Philip ABBET committed
301
302
        if not self._name:
            raise RuntimeError("library has no name")
303

Philip ABBET's avatar
Philip ABBET committed
304
        return self.storage.hash()
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320

    def json_dumps(self, indent=4):
        """Dumps the JSON declaration of this object in a string


        Parameters:

          indent (int): The number of indentation spaces at every indentation level


        Returns:

          str: The JSON representation for this object

        """

Samuel GAIST's avatar
Samuel GAIST committed
321
        return simplejson.dumps(self.data, indent=indent, cls=utils.NumpyJSONEncoder)
322
323
324
325
326
327
328
329
330

    def __str__(self):
        return self.json_dumps()

    def write(self, storage=None):
        """Writes contents to prefix location.

        Parameters:

331
332
333
          storage (:py:class:`.Storage`, Optional): If you pass a new
            storage, then this object will be written to that storage point
            rather than its default.
334
335
336

        """

Samuel GAIST's avatar
Samuel GAIST committed
337
        if self.data["language"] == "unknown":
338
339
340
341
342
            raise RuntimeError("library has no programming language set")

        if storage is None:
            if not self._name:
                raise RuntimeError("library has no name")
Samuel GAIST's avatar
Samuel GAIST committed
343
            storage = self.storage  # overwrite
344
345
346
347
348
349
350
351
352
353
354

        storage.save(str(self), self.code, self.description)

    def export(self, prefix):
        """Recursively exports itself into another prefix

        Other required libraries are also copied.


        Parameters:

Samuel GAIST's avatar
Samuel GAIST committed
355
          prefix (str): Establishes the prefix of your installation.
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374


        Returns:

          None


        Raises:

          RuntimeError: If prefix and self.prefix point to the same directory.

        """

        if not self._name:
            raise RuntimeError("library has no name")

        if not self.valid:
            raise RuntimeError("library is not valid")

Samuel GAIST's avatar
Samuel GAIST committed
375
        if prefix == self.prefix:
Samuel GAIST's avatar
Samuel GAIST committed
376
377
378
            raise RuntimeError(
                "Cannot export library to the same prefix (" "%s)" % (prefix)
            )
379
380
381
382
383

        for k in self.libraries.values():
            k.export(prefix)

        self.write(Storage(prefix, self.name))