utils.py 12.3 KB
Newer Older
Philip ABBET's avatar
Philip ABBET committed
1
2
3
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

Samuel GAIST's avatar
Samuel GAIST committed
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
###################################################################################
#                                                                                 #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/               #
# Contact: beat.support@idiap.ch                                                  #
#                                                                                 #
# Redistribution and use in source and binary forms, with or without              #
# modification, are permitted provided that the following conditions are met:     #
#                                                                                 #
# 1. Redistributions of source code must retain the above copyright notice, this  #
# list of conditions and the following disclaimer.                                #
#                                                                                 #
# 2. Redistributions in binary form must reproduce the above copyright notice,    #
# this list of conditions and the following disclaimer in the documentation       #
# and/or other materials provided with the distribution.                          #
#                                                                                 #
# 3. Neither the name of the copyright holder nor the names of its contributors   #
# may be used to endorse or promote products derived from this software without   #
# specific prior written permission.                                              #
#                                                                                 #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED   #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE          #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE    #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL      #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR      #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER      #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,   #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE   #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.            #
#                                                                                 #
###################################################################################

Philip ABBET's avatar
Philip ABBET committed
36

Samuel GAIST's avatar
Samuel GAIST committed
37
38
39
40
41
42
43
"""
=====
utils
=====

This module implements helper classes and functions.
"""
Philip ABBET's avatar
Philip ABBET committed
44
45
46
47

import os
import shutil
import collections
Philip ABBET's avatar
Philip ABBET committed
48
import numpy
Philip ABBET's avatar
Philip ABBET committed
49
50
51
52
53
54
import simplejson
import six

from . import hash


Samuel GAIST's avatar
Samuel GAIST committed
55
# ----------------------------------------------------------
56

57

Samuel GAIST's avatar
Samuel GAIST committed
58
def hashed_or_simple(prefix, what, path, suffix=".json"):
Samuel GAIST's avatar
Samuel GAIST committed
59
60
    """Returns a hashed path or simple path depending on where the resource is
    """
61

Samuel GAIST's avatar
Samuel GAIST committed
62
    username, right_bit = path.split("/", 1)
Philip ABBET's avatar
Philip ABBET committed
63
    hashed_prefix = hash.toUserPath(username)
Samuel GAIST's avatar
Samuel GAIST committed
64
    candidate = os.path.join(prefix, what, hashed_prefix, right_bit) + suffix
65

66
    if os.path.exists(candidate):
Philip ABBET's avatar
Philip ABBET committed
67
        return candidate
68

Samuel GAIST's avatar
Samuel GAIST committed
69
    return os.path.join(prefix, what, path + suffix)
70
71


Samuel GAIST's avatar
Samuel GAIST committed
72
# ----------------------------------------------------------
Philip ABBET's avatar
Philip ABBET committed
73

74

Philip ABBET's avatar
Philip ABBET committed
75
def safe_rmfile(f):
Philip ABBET's avatar
Philip ABBET committed
76
    """Safely removes a file from the disk"""
Philip ABBET's avatar
Philip ABBET committed
77

78
79
    if os.path.exists(f):
        os.unlink(f)
Philip ABBET's avatar
Philip ABBET committed
80
81


Samuel GAIST's avatar
Samuel GAIST committed
82
# ----------------------------------------------------------
Philip ABBET's avatar
Philip ABBET committed
83

Philip ABBET's avatar
Philip ABBET committed
84
85

def safe_rmdir(f):
Philip ABBET's avatar
Philip ABBET committed
86
    """Safely removes the directory containg a given file from the disk"""
Philip ABBET's avatar
Philip ABBET committed
87

Philip ABBET's avatar
Philip ABBET committed
88
    d = os.path.dirname(f)
89
90
91
92
93
94

    if not os.path.exists(d):
        return

    if not os.listdir(d):
        os.rmdir(d)
Philip ABBET's avatar
Philip ABBET committed
95
96


Samuel GAIST's avatar
Samuel GAIST committed
97
# ----------------------------------------------------------
Philip ABBET's avatar
Philip ABBET committed
98

Philip ABBET's avatar
Philip ABBET committed
99
100

def extension_for_language(language):
Philip ABBET's avatar
Philip ABBET committed
101
    """Returns the preferred extension for a given programming language
Philip ABBET's avatar
Philip ABBET committed
102

Philip ABBET's avatar
Philip ABBET committed
103
104
    The set of languages supported must match those declared in our
    ``common.json`` schema.
Philip ABBET's avatar
Philip ABBET committed
105

Philip ABBET's avatar
Philip ABBET committed
106
    Parameters:
Philip ABBET's avatar
Philip ABBET committed
107

Philip ABBET's avatar
Philip ABBET committed
108
      language (str) The language for which you'd like to get the extension for.
Philip ABBET's avatar
Philip ABBET committed
109
110


Philip ABBET's avatar
Philip ABBET committed
111
    Returns:
Philip ABBET's avatar
Philip ABBET committed
112

Philip ABBET's avatar
Philip ABBET committed
113
      str: The extension for the given language, including a leading ``.`` (dot)
Philip ABBET's avatar
Philip ABBET committed
114
115


Philip ABBET's avatar
Philip ABBET committed
116
    Raises:
Philip ABBET's avatar
Philip ABBET committed
117

Philip ABBET's avatar
Philip ABBET committed
118
      KeyError: If the language is not defined in our internal dictionary.
Philip ABBET's avatar
Philip ABBET committed
119

Philip ABBET's avatar
Philip ABBET committed
120
    """
Philip ABBET's avatar
Philip ABBET committed
121

Samuel GAIST's avatar
Samuel GAIST committed
122
    return dict(unknown="", cxx=".so", matlab=".m", python=".py", r=".r")[language]
123
124


Samuel GAIST's avatar
Samuel GAIST committed
125
# ----------------------------------------------------------
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146


class Prefix(object):
    def __init__(self, paths=None):
        if isinstance(paths, list):
            self.paths = paths
        elif paths is not None:
            self.paths = [paths]
        else:
            self.paths = []

    def add(self, path):
        self.paths.append(path)

    def path(self, filename):
        for p in self.paths:
            fullpath = os.path.join(p, filename)
            if os.path.exists(fullpath):
                return fullpath

        return os.path.join(self.paths[0], filename)
Philip ABBET's avatar
Philip ABBET committed
147
148


Samuel GAIST's avatar
Samuel GAIST committed
149
# ----------------------------------------------------------
Philip ABBET's avatar
Philip ABBET committed
150

Philip ABBET's avatar
Philip ABBET committed
151
152

class File(object):
Philip ABBET's avatar
Philip ABBET committed
153
    """User helper to read and write file objects"""
Philip ABBET's avatar
Philip ABBET committed
154

Philip ABBET's avatar
Philip ABBET committed
155
    def __init__(self, path, binary=False):
Philip ABBET's avatar
Philip ABBET committed
156

Philip ABBET's avatar
Philip ABBET committed
157
158
        self.path = path
        self.binary = binary
Philip ABBET's avatar
Philip ABBET committed
159

Philip ABBET's avatar
Philip ABBET committed
160
    def exists(self):
Philip ABBET's avatar
Philip ABBET committed
161

Philip ABBET's avatar
Philip ABBET committed
162
        return os.path.exists(self.path)
Philip ABBET's avatar
Philip ABBET committed
163

Philip ABBET's avatar
Philip ABBET committed
164
    def load(self):
Philip ABBET's avatar
Philip ABBET committed
165

Samuel GAIST's avatar
Samuel GAIST committed
166
        mode = "rb" if self.binary else "rt"
167
168
        with open(self.path, mode) as f:
            return f.read()
Philip ABBET's avatar
Philip ABBET committed
169

Philip ABBET's avatar
Philip ABBET committed
170
    def try_load(self):
Philip ABBET's avatar
Philip ABBET committed
171

Philip ABBET's avatar
Philip ABBET committed
172
173
174
        if os.path.exists(self.path):
            return self.load()
        return None
Philip ABBET's avatar
Philip ABBET committed
175

Philip ABBET's avatar
Philip ABBET committed
176
    def backup(self):
Philip ABBET's avatar
Philip ABBET committed
177

178
        if not os.path.exists(self.path):
Samuel GAIST's avatar
Samuel GAIST committed
179
            return  # no point in backing-up
180

Samuel GAIST's avatar
Samuel GAIST committed
181
        backup = self.path + "~"
182
183
184
        if os.path.exists(backup):
            os.remove(backup)

Philip ABBET's avatar
Philip ABBET committed
185
        shutil.copy(self.path, backup)
Philip ABBET's avatar
Philip ABBET committed
186

Philip ABBET's avatar
Philip ABBET committed
187
    def save(self, contents):
Philip ABBET's avatar
Philip ABBET committed
188

Philip ABBET's avatar
Philip ABBET committed
189
        d = os.path.dirname(self.path)
190
191
        if not os.path.exists(d):
            os.makedirs(d)
Philip ABBET's avatar
Philip ABBET committed
192

193
194
        if os.path.exists(self.path):
            self.backup()
Philip ABBET's avatar
Philip ABBET committed
195

Samuel GAIST's avatar
Samuel GAIST committed
196
        mode = "wb" if self.binary else "wt"
197
        if self.binary:
Samuel GAIST's avatar
Samuel GAIST committed
198
            mode = "wb"
199
            if isinstance(contents, six.string_types):
Samuel GAIST's avatar
Samuel GAIST committed
200
                contents = contents.encode("utf-8")
201
        else:
Samuel GAIST's avatar
Samuel GAIST committed
202
            mode = "wt"
203
            if not isinstance(contents, six.string_types):
Samuel GAIST's avatar
Samuel GAIST committed
204
                contents = contents.decode("utf-8")
205

206
207
        with open(self.path, mode) as f:
            f.write(contents)
Philip ABBET's avatar
Philip ABBET committed
208

Philip ABBET's avatar
Philip ABBET committed
209
    def remove(self):
Philip ABBET's avatar
Philip ABBET committed
210

Philip ABBET's avatar
Philip ABBET committed
211
        safe_rmfile(self.path)
Samuel GAIST's avatar
Samuel GAIST committed
212
213
        safe_rmfile(self.path + "~")  # backup
        safe_rmdir(self.path)  # remove containing directory
Philip ABBET's avatar
Philip ABBET committed
214
215


Samuel GAIST's avatar
Samuel GAIST committed
216
# ----------------------------------------------------------
Philip ABBET's avatar
Philip ABBET committed
217

Philip ABBET's avatar
Philip ABBET committed
218

219
class AbstractStorage(object):
Philip ABBET's avatar
Philip ABBET committed
220

221
222
223
    asset_type = None
    asset_folder = None

Philip ABBET's avatar
Philip ABBET committed
224
    def __init__(self, path):
Philip ABBET's avatar
Philip ABBET committed
225

226
227
228
        if not all(
            [type(attr) == str for attr in [self.asset_type, self.asset_folder]]
        ):
229
230
231
232
233
            raise TypeError(
                "asset_type and asset_folder must be configured properly\n"
                "asset_type: {}\n"
                "asset_folder: {}".format(self.asset_type, self.asset_folder)
            )
234

Philip ABBET's avatar
Philip ABBET committed
235
        self.path = path
Samuel GAIST's avatar
Samuel GAIST committed
236
237
        self.json = File(self.path + ".json")
        self.doc = File(self.path + ".rst")
Philip ABBET's avatar
Philip ABBET committed
238

Philip ABBET's avatar
Philip ABBET committed
239
240
    def exists(self):
        """If the database declaration file exists"""
241

Philip ABBET's avatar
Philip ABBET committed
242
        return self.json.exists()
Philip ABBET's avatar
Philip ABBET committed
243

244
245
246
247
248
249
250
251
252
253
254
    def remove(self):
        """Removes the object from the disk"""

        self.json.remove()
        self.doc.remove()

    def hash(self):
        """The 64-character hash of the database declaration JSON"""

        raise NotImplementedError

Philip ABBET's avatar
Philip ABBET committed
255
256
    def load(self):
        """Loads the JSON declaration as a file"""
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279

        raise NotImplementedError

    def save(self):
        """Saves the JSON declaration as files"""

        raise NotImplementedError


class Storage(AbstractStorage):
    """Resolves paths for objects that provide only a description"""

    def __init__(self, path):
        super(Storage, self).__init__(path)

    def hash(self, description="description"):
        """Re-imp"""

        return hash.hashJSONFile(self.json.path, description)

    def load(self):
        """Re-imp"""

Samuel GAIST's avatar
Samuel GAIST committed
280
        tp = collections.namedtuple("Storage", ["declaration", "description"])
Philip ABBET's avatar
Philip ABBET committed
281
        return tp(self.json.load(), self.doc.try_load())
Philip ABBET's avatar
Philip ABBET committed
282

Philip ABBET's avatar
Philip ABBET committed
283
    def save(self, declaration, description=None):
284
285
        """Re-imp"""

Samuel GAIST's avatar
Samuel GAIST committed
286
287
        if description:
            self.doc.save(description.encode("utf8"))
Philip ABBET's avatar
Philip ABBET committed
288
289
290
        if not isinstance(declaration, six.string_types):
            declaration = simplejson.dumps(declaration, indent=4)
        self.json.save(declaration)
Philip ABBET's avatar
Philip ABBET committed
291
292


Samuel GAIST's avatar
Samuel GAIST committed
293
# ----------------------------------------------------------
Philip ABBET's avatar
Philip ABBET committed
294

Philip ABBET's avatar
Philip ABBET committed
295

296
class CodeStorage(AbstractStorage):
Philip ABBET's avatar
Philip ABBET committed
297
298
299
300
301
302
303
304
305
    """Resolves paths for objects that provide a description and code

    Parameters:

      language (str): One of the valdid programming languages

    """

    def __init__(self, path, language=None):
306
        super(CodeStorage, self).__init__(path)
Philip ABBET's avatar
Philip ABBET committed
307
308

        self._language = language or self.__auto_discover_language()
Samuel GAIST's avatar
Samuel GAIST committed
309
310
311
        self.code = File(
            self.path + extension_for_language(self._language), binary=True
        )
Philip ABBET's avatar
Philip ABBET committed
312
313
314
315
316
317

    def __auto_discover_language(self, json=None):
        """Discovers and sets the language from its own JSON descriptor"""
        try:
            text = json or self.json.load()
            json = simplejson.loads(text)
Samuel GAIST's avatar
Samuel GAIST committed
318
            return json["language"]
319
        except (IOError, KeyError, simplejson.JSONDecodeError):
Samuel GAIST's avatar
Samuel GAIST committed
320
            return "unknown"
Philip ABBET's avatar
Philip ABBET committed
321
322
323
324
325
326
327
328

    @property
    def language(self):
        return self._language

    @language.setter
    def language(self, value):
        self._language = value
Samuel GAIST's avatar
Samuel GAIST committed
329
330
331
        self.code = File(
            self.path + extension_for_language(self._language), binary=True
        )
Philip ABBET's avatar
Philip ABBET committed
332
333

    def hash(self):
334
        """Re-imp"""
Philip ABBET's avatar
Philip ABBET committed
335
336

        if self.code.exists():
Samuel GAIST's avatar
Samuel GAIST committed
337
338
339
340
341
342
            return hash.hash(
                dict(
                    json=hash.hashJSONFile(self.json.path, "description"),
                    code=hash.hashFileContents(self.code.path),
                )
            )
Philip ABBET's avatar
Philip ABBET committed
343
        else:
Samuel GAIST's avatar
Samuel GAIST committed
344
345
346
            return hash.hash(
                dict(json=hash.hashJSONFile(self.json.path, "description"))
            )
Philip ABBET's avatar
Philip ABBET committed
347
348

    def exists(self):
349
350
351
        """Re-imp"""

        return super(CodeStorage, self).exists() and self.code.exists()
Philip ABBET's avatar
Philip ABBET committed
352
353

    def load(self):
354
355
        """Re-imp"""

Samuel GAIST's avatar
Samuel GAIST committed
356
357
358
        tp = collections.namedtuple(
            "CodeStorage", ["declaration", "code", "description"]
        )
Philip ABBET's avatar
Philip ABBET committed
359
360
361
        return tp(self.json.load(), self.code.try_load(), self.doc.try_load())

    def save(self, declaration, code=None, description=None):
362
363
        """Re-imp"""

Philip ABBET's avatar
Philip ABBET committed
364
        if description:
Samuel GAIST's avatar
Samuel GAIST committed
365
            self.doc.save(description.encode("utf8"))
Philip ABBET's avatar
Philip ABBET committed
366
367
368
369
370
371

        if not isinstance(declaration, six.string_types):
            declaration = simplejson.dumps(declaration, indent=4)
        self.json.save(declaration)

        if code:
Samuel GAIST's avatar
Samuel GAIST committed
372
            if self._language == "unknown":
Philip ABBET's avatar
Philip ABBET committed
373
374
375
376
                self.language = self.__auto_discover_language(declaration)
            self.code.save(code)

    def remove(self):
377
378
379
        """Re-imp"""

        super(CodeStorage, self).remove()
Philip ABBET's avatar
Philip ABBET committed
380
        self.code.remove()
381
382


Samuel GAIST's avatar
Samuel GAIST committed
383
# ----------------------------------------------------------
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398


class NumpyJSONEncoder(simplejson.JSONEncoder):
    """Encodes numpy arrays and scalars

    See Also:

      :py:class:`simplejson.JSONEncoder`

    """

    def default(self, obj):
        if isinstance(obj, numpy.ndarray) or isinstance(obj, numpy.generic):
            return obj.tolist()
        elif isinstance(obj, numpy.dtype):
Samuel GAIST's avatar
Samuel GAIST committed
399
400
            if obj.name == "str":
                return "string"
401
402
            return obj.name
        return simplejson.JSONEncoder.default(self, obj)
403
404
405
406
407


# ----------------------------------------------------------


408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
def error_on_duplicate_key_hook(pairs):
    """JSON loader hook that will error out if several same keys are found

    Returns an OrderedDict if everything goes well
    """

    dct = collections.OrderedDict()
    for key, value in pairs:
        if key in dct:
            raise RuntimeError(
                "Invalid file content\n{} found several times".format(key)
            )
        dct[key] = value

    return dct


# ----------------------------------------------------------


428
429
430
def has_argument(method, argument):
    try:
        from inspect import signature
Samuel GAIST's avatar
Samuel GAIST committed
431

432
433
434
435
        sig = signature(method)
        params = sig.parameters
    except ImportError:
        from inspect import getargspec
Samuel GAIST's avatar
Samuel GAIST committed
436

437
438
439
        params = getargspec(method).args

    return argument in params