library.py 12 KB
Newer Older
André Anjos's avatar
André Anjos committed
1
2
3
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

Samuel GAIST's avatar
Samuel GAIST committed
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
###################################################################################
#                                                                                 #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/               #
# Contact: beat.support@idiap.ch                                                  #
#                                                                                 #
# Redistribution and use in source and binary forms, with or without              #
# modification, are permitted provided that the following conditions are met:     #
#                                                                                 #
# 1. Redistributions of source code must retain the above copyright notice, this  #
# list of conditions and the following disclaimer.                                #
#                                                                                 #
# 2. Redistributions in binary form must reproduce the above copyright notice,    #
# this list of conditions and the following disclaimer in the documentation       #
# and/or other materials provided with the distribution.                          #
#                                                                                 #
# 3. Neither the name of the copyright holder nor the names of its contributors   #
# may be used to endorse or promote products derived from this software without   #
# specific prior written permission.                                              #
#                                                                                 #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED   #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE          #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE    #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL      #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR      #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER      #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,   #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE   #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.            #
#                                                                                 #
###################################################################################
André Anjos's avatar
André Anjos committed
35
36


37
38
39
40
41
42
43
"""
=======
library
=======

Validation for libraries
"""
André Anjos's avatar
André Anjos committed
44
45

import os
46
import simplejson as json
André Anjos's avatar
André Anjos committed
47
48

from . import loader
49
50
51
from . import utils


52
# ----------------------------------------------------------
53

54
55

class Storage(utils.CodeStorage):
Philip ABBET's avatar
Philip ABBET committed
56
    """Resolves paths for libraries
57

Philip ABBET's avatar
Philip ABBET committed
58
    Parameters:
59

Samuel GAIST's avatar
Samuel GAIST committed
60
      prefix (str): Establishes the prefix of
61
        your installation.
62

Philip ABBET's avatar
Philip ABBET committed
63
64
      name (str): The name of the library object in the format
        ``<user>/<name>/<version>``.
65

Philip ABBET's avatar
Philip ABBET committed
66
    """
67

68
69
70
    asset_type = "library"
    asset_folder = "libraries"

Philip ABBET's avatar
Philip ABBET committed
71
    def __init__(self, prefix, name, language=None):
72

Samuel GAIST's avatar
Samuel GAIST committed
73
        if name.count("/") != 2:
Philip ABBET's avatar
Philip ABBET committed
74
            raise RuntimeError("invalid library name: `%s'" % name)
75

Samuel GAIST's avatar
Samuel GAIST committed
76
        self.username, self.name, self.version = name.split("/")
Philip ABBET's avatar
Philip ABBET committed
77
        self.fullname = name
Samuel GAIST's avatar
Samuel GAIST committed
78
        self.prefix = prefix
79

80
81
82
        path = utils.hashed_or_simple(
            self.prefix, self.asset_folder, name, suffix=".json"
        )
83
        path = path[:-5]
Philip ABBET's avatar
Philip ABBET committed
84
        super(Storage, self).__init__(path, language)
85

André Anjos's avatar
André Anjos committed
86

87
# ----------------------------------------------------------
88

André Anjos's avatar
André Anjos committed
89
90

class Library(object):
Philip ABBET's avatar
Philip ABBET committed
91
    """Librarys represent independent algorithm components within the platform.
André Anjos's avatar
André Anjos committed
92

Philip ABBET's avatar
Philip ABBET committed
93
94
95
    This class can only parse the meta-parameters of the library. The actual
    library is not directly treated by this class - only by the associated
    algorithms.
André Anjos's avatar
André Anjos committed
96
97


Philip ABBET's avatar
Philip ABBET committed
98
    Parameters:
André Anjos's avatar
André Anjos committed
99

Samuel GAIST's avatar
Samuel GAIST committed
100
      prefix (str): Establishes the prefix of your installation.
André Anjos's avatar
André Anjos committed
101

Philip ABBET's avatar
Philip ABBET committed
102
      name (str): The fully qualified algorithm name (e.g. ``user/algo/1``)
André Anjos's avatar
André Anjos committed
103

104
105
106
107
      library_cache (:py:class:`dict`, Optional): A dictionary mapping library
        names to loaded libraries. This parameter is optional and, if passed,
        may greatly speed-up library loading times as libraries that are
        already loaded may be re-used.
André Anjos's avatar
André Anjos committed
108
109


Philip ABBET's avatar
Philip ABBET committed
110
    Attributes:
André Anjos's avatar
André Anjos committed
111

Philip ABBET's avatar
Philip ABBET committed
112
      name (str): The library name
André Anjos's avatar
André Anjos committed
113

Philip ABBET's avatar
Philip ABBET committed
114
115
      description (str): The short description string, loaded from the JSON
        file if one was set.
116

Philip ABBET's avatar
Philip ABBET committed
117
      documentation (str): The full-length docstring for this object.
118

Philip ABBET's avatar
Philip ABBET committed
119
120
      storage (object): A simple object that provides information about file
        paths for this library
121

Philip ABBET's avatar
Philip ABBET committed
122
123
      libraries (dict): A mapping object defining other libraries this library
        needs to load so it can work properly.
André Anjos's avatar
André Anjos committed
124

Philip ABBET's avatar
Philip ABBET committed
125
126
      uses (dict): A mapping object defining the required library import name
        (keys) and the full-names (values).
André Anjos's avatar
André Anjos committed
127

Philip ABBET's avatar
Philip ABBET committed
128
129
      errors (list): A list containing errors found while loading this
        library.
130

Philip ABBET's avatar
Philip ABBET committed
131
132
      data (dict): The original data for this library, as loaded by our JSON
        decoder.
André Anjos's avatar
André Anjos committed
133

Philip ABBET's avatar
Philip ABBET committed
134
135
      code (str): The code that is associated with this library, loaded as a
        text (or binary) file.
André Anjos's avatar
André Anjos committed
136

Philip ABBET's avatar
Philip ABBET committed
137
    """
André Anjos's avatar
André Anjos committed
138

Philip ABBET's avatar
Philip ABBET committed
139
    def __init__(self, prefix, name, library_cache=None):
André Anjos's avatar
André Anjos committed
140

Philip ABBET's avatar
Philip ABBET committed
141
142
143
144
145
        self._name = None
        self.storage = None
        self.prefix = prefix
        self.errors = []
        self.libraries = {}
André Anjos's avatar
André Anjos committed
146

Philip ABBET's avatar
Philip ABBET committed
147
        library_cache = library_cache if library_cache is not None else {}
André Anjos's avatar
André Anjos committed
148

Philip ABBET's avatar
Philip ABBET committed
149
150
151
        try:
            self._load(name, library_cache)
        finally:
Samuel GAIST's avatar
Samuel GAIST committed
152
            if self._name is not None:  # registers it into the cache, even if failed
Philip ABBET's avatar
Philip ABBET committed
153
                library_cache[self._name] = self
154

Philip ABBET's avatar
Philip ABBET committed
155
156
    def _load(self, data, library_cache):
        """Loads the library"""
157

Philip ABBET's avatar
Philip ABBET committed
158
        self._name = data
159

Philip ABBET's avatar
Philip ABBET committed
160
161
162
        self.storage = Storage(self.prefix, data)
        json_path = self.storage.json.path
        if not self.storage.exists():
Samuel GAIST's avatar
Samuel GAIST committed
163
            self.errors.append("Library declaration file not found: %s" % json_path)
Philip ABBET's avatar
Philip ABBET committed
164
            return
165

Samuel GAIST's avatar
Samuel GAIST committed
166
        with open(json_path, "rb") as f:
167
168
169
170
171
172
173
174
            try:
                self.data = json.loads(
                    f.read().decode("utf-8"),
                    object_pairs_hook=utils.error_on_duplicate_key_hook,
                )
            except RuntimeError as error:
                self.errors.append("Library declaration file invalid: %s" % error)
                return
André Anjos's avatar
André Anjos committed
175

Philip ABBET's avatar
Philip ABBET committed
176
        self.code_path = self.storage.code.path
André Anjos's avatar
André Anjos committed
177

Philip ABBET's avatar
Philip ABBET committed
178
        # if no errors so far, make sense out of the library data
Samuel GAIST's avatar
Samuel GAIST committed
179
        self.data.setdefault("uses", {})
André Anjos's avatar
André Anjos committed
180

Philip ABBET's avatar
Philip ABBET committed
181
182
183
        if self.uses is not None:
            for name, value in self.uses.items():
                self.libraries[value] = Library(self.prefix, value, library_cache)
André Anjos's avatar
André Anjos committed
184

Philip ABBET's avatar
Philip ABBET committed
185
        self.libraries[self._name] = self
André Anjos's avatar
André Anjos committed
186

Philip ABBET's avatar
Philip ABBET committed
187
188
    def uses_dict(self):
        """Returns the usage dictionary for all dependent modules"""
André Anjos's avatar
André Anjos committed
189

Samuel GAIST's avatar
Samuel GAIST committed
190
        if self.data["language"] == "unknown":
Philip ABBET's avatar
Philip ABBET committed
191
            raise RuntimeError("library has no programming language set")
192

Philip ABBET's avatar
Philip ABBET committed
193
194
        if not self._name:
            raise RuntimeError("library has no name")
195

Philip ABBET's avatar
Philip ABBET committed
196
        retval = {}
André Anjos's avatar
André Anjos committed
197

Philip ABBET's avatar
Philip ABBET committed
198
        if self.uses is not None:
André Anjos's avatar
André Anjos committed
199

Philip ABBET's avatar
Philip ABBET committed
200
201
            for name, value in self.uses.items():
                retval[name] = dict(
Samuel GAIST's avatar
Samuel GAIST committed
202
203
204
                    path=self.libraries[value].storage.code.path,
                    uses=self.libraries[value].uses_dict(),
                )
André Anjos's avatar
André Anjos committed
205

Philip ABBET's avatar
Philip ABBET committed
206
        return retval
André Anjos's avatar
André Anjos committed
207

Philip ABBET's avatar
Philip ABBET committed
208
209
    def load(self):
        """Loads the Python module for this library resolving all references
André Anjos's avatar
André Anjos committed
210

Philip ABBET's avatar
Philip ABBET committed
211
212
        Returns the loaded Python module.
        """
André Anjos's avatar
André Anjos committed
213

Samuel GAIST's avatar
Samuel GAIST committed
214
        if self.data["language"] == "unknown":
Philip ABBET's avatar
Philip ABBET committed
215
            raise RuntimeError("library has no programming language set")
216

Philip ABBET's avatar
Philip ABBET committed
217
218
        if not self._name:
            raise RuntimeError("library has no name")
219

Samuel GAIST's avatar
Samuel GAIST committed
220
221
222
        return loader.load_module(
            self.name.replace(os.sep, "_"), self.storage.code.path, self.uses_dict()
        )
223

Philip ABBET's avatar
Philip ABBET committed
224
225
    @property
    def name(self):
226
227
        """Returns the name of this object"""

Samuel GAIST's avatar
Samuel GAIST committed
228
        return self._name or "__unnamed_library__"
229

Philip ABBET's avatar
Philip ABBET committed
230
231
    @name.setter
    def name(self, value):
232

Samuel GAIST's avatar
Samuel GAIST committed
233
        if self.data["language"] == "unknown":
Philip ABBET's avatar
Philip ABBET committed
234
            raise RuntimeError("library has no programming language set")
235

Philip ABBET's avatar
Philip ABBET committed
236
        self._name = value
Samuel GAIST's avatar
Samuel GAIST committed
237
        self.storage = Storage(self.prefix, value, self.data["language"])
André Anjos's avatar
André Anjos committed
238

Philip ABBET's avatar
Philip ABBET committed
239
240
241
    @property
    def schema_version(self):
        """Returns the schema version"""
Samuel GAIST's avatar
Samuel GAIST committed
242
        return self.data.get("schema_version", 1)
André Anjos's avatar
André Anjos committed
243

Philip ABBET's avatar
Philip ABBET committed
244
245
246
    @property
    def language(self):
        """Returns the current language set for the library code"""
Samuel GAIST's avatar
Samuel GAIST committed
247
        return self.data["language"]
248

Philip ABBET's avatar
Philip ABBET committed
249
250
251
    @language.setter
    def language(self, value):
        """Sets the current executable code programming language"""
Samuel GAIST's avatar
Samuel GAIST committed
252
253
254
        if self.storage:
            self.storage.language = value
        self.data["language"] = value
Philip ABBET's avatar
Philip ABBET committed
255
        self._check_language_consistence()
256

Philip ABBET's avatar
Philip ABBET committed
257
258
259
    @property
    def valid(self):
        """A boolean that indicates if this library is valid or not"""
260

Philip ABBET's avatar
Philip ABBET committed
261
        return not bool(self.errors)
262

Philip ABBET's avatar
Philip ABBET committed
263
264
    @property
    def uses(self):
Samuel GAIST's avatar
Samuel GAIST committed
265
        return self.data.get("uses")
266

Philip ABBET's avatar
Philip ABBET committed
267
268
    @uses.setter
    def uses(self, value):
Samuel GAIST's avatar
Samuel GAIST committed
269
        self.data["uses"] = value
Philip ABBET's avatar
Philip ABBET committed
270
        return value
271

Philip ABBET's avatar
Philip ABBET committed
272
273
274
    @property
    def description(self):
        """The short description for this object"""
Samuel GAIST's avatar
Samuel GAIST committed
275
        return self.data.get("description", None)
276

Philip ABBET's avatar
Philip ABBET committed
277
278
279
    @description.setter
    def description(self, value):
        """Sets the short description for this object"""
Samuel GAIST's avatar
Samuel GAIST committed
280
        self.data["description"] = value
281

Philip ABBET's avatar
Philip ABBET committed
282
283
284
    @property
    def documentation(self):
        """The full-length description for this object"""
285

Philip ABBET's avatar
Philip ABBET committed
286
287
        if not self._name:
            raise RuntimeError("library has no name")
288

Philip ABBET's avatar
Philip ABBET committed
289
290
291
        if self.storage.doc.exists():
            return self.storage.doc.load()
        return None
292

Philip ABBET's avatar
Philip ABBET committed
293
294
295
    @documentation.setter
    def documentation(self, value):
        """Sets the full-length description for this object"""
296

Philip ABBET's avatar
Philip ABBET committed
297
298
        if not self._name:
            raise RuntimeError("library has no name")
299

Samuel GAIST's avatar
Samuel GAIST committed
300
        if hasattr(value, "read"):
Philip ABBET's avatar
Philip ABBET committed
301
302
303
            self.storage.doc.save(value.read())
        else:
            self.storage.doc.save(value)
304

Philip ABBET's avatar
Philip ABBET committed
305
306
    def hash(self):
        """Returns the hexadecimal hash for the current library"""
307

Philip ABBET's avatar
Philip ABBET committed
308
309
        if not self._name:
            raise RuntimeError("library has no name")
310

Philip ABBET's avatar
Philip ABBET committed
311
        return self.storage.hash()
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327

    def json_dumps(self, indent=4):
        """Dumps the JSON declaration of this object in a string


        Parameters:

          indent (int): The number of indentation spaces at every indentation level


        Returns:

          str: The JSON representation for this object

        """

328
        return json.dumps(self.data, indent=indent, cls=utils.NumpyJSONEncoder)
329
330
331
332
333
334
335
336
337

    def __str__(self):
        return self.json_dumps()

    def write(self, storage=None):
        """Writes contents to prefix location.

        Parameters:

338
339
340
          storage (:py:class:`.Storage`, Optional): If you pass a new
            storage, then this object will be written to that storage point
            rather than its default.
341
342
343

        """

Samuel GAIST's avatar
Samuel GAIST committed
344
        if self.data["language"] == "unknown":
345
346
347
348
349
            raise RuntimeError("library has no programming language set")

        if storage is None:
            if not self._name:
                raise RuntimeError("library has no name")
Samuel GAIST's avatar
Samuel GAIST committed
350
            storage = self.storage  # overwrite
351
352
353
354
355
356
357
358
359
360
361

        storage.save(str(self), self.code, self.description)

    def export(self, prefix):
        """Recursively exports itself into another prefix

        Other required libraries are also copied.


        Parameters:

Samuel GAIST's avatar
Samuel GAIST committed
362
          prefix (str): Establishes the prefix of your installation.
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381


        Returns:

          None


        Raises:

          RuntimeError: If prefix and self.prefix point to the same directory.

        """

        if not self._name:
            raise RuntimeError("library has no name")

        if not self.valid:
            raise RuntimeError("library is not valid")

Samuel GAIST's avatar
Samuel GAIST committed
382
        if prefix == self.prefix:
Samuel GAIST's avatar
Samuel GAIST committed
383
384
385
            raise RuntimeError(
                "Cannot export library to the same prefix (" "%s)" % (prefix)
            )
386
387
388
389
390

        for k in self.libraries.values():
            k.export(prefix)

        self.write(Storage(prefix, self.name))