utils.py 11.6 KB
Newer Older
Philip ABBET's avatar
Philip ABBET committed
1
2
3
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

Samuel GAIST's avatar
Samuel GAIST committed
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
###################################################################################
#                                                                                 #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/               #
# Contact: beat.support@idiap.ch                                                  #
#                                                                                 #
# Redistribution and use in source and binary forms, with or without              #
# modification, are permitted provided that the following conditions are met:     #
#                                                                                 #
# 1. Redistributions of source code must retain the above copyright notice, this  #
# list of conditions and the following disclaimer.                                #
#                                                                                 #
# 2. Redistributions in binary form must reproduce the above copyright notice,    #
# this list of conditions and the following disclaimer in the documentation       #
# and/or other materials provided with the distribution.                          #
#                                                                                 #
# 3. Neither the name of the copyright holder nor the names of its contributors   #
# may be used to endorse or promote products derived from this software without   #
# specific prior written permission.                                              #
#                                                                                 #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED   #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE          #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE    #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL      #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR      #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER      #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,   #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE   #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.            #
#                                                                                 #
###################################################################################

Philip ABBET's avatar
Philip ABBET committed
36

Samuel GAIST's avatar
Samuel GAIST committed
37
38
39
40
41
42
43
"""
=====
utils
=====

This module implements helper classes and functions.
"""
Philip ABBET's avatar
Philip ABBET committed
44
45
46
47

import os
import shutil
import collections
Philip ABBET's avatar
Philip ABBET committed
48
import numpy
Philip ABBET's avatar
Philip ABBET committed
49
50
51
52
53
54
import simplejson
import six

from . import hash


Samuel GAIST's avatar
Samuel GAIST committed
55
# ----------------------------------------------------------
56

57

Samuel GAIST's avatar
Samuel GAIST committed
58
def hashed_or_simple(prefix, what, path, suffix=".json"):
Samuel GAIST's avatar
Samuel GAIST committed
59
60
    """Returns a hashed path or simple path depending on where the resource is
    """
61

Samuel GAIST's avatar
Samuel GAIST committed
62
    username, right_bit = path.split("/", 1)
Philip ABBET's avatar
Philip ABBET committed
63
    hashed_prefix = hash.toUserPath(username)
Samuel GAIST's avatar
Samuel GAIST committed
64
    candidate = os.path.join(prefix, what, hashed_prefix, right_bit) + suffix
65

66
    if os.path.exists(candidate):
Philip ABBET's avatar
Philip ABBET committed
67
        return candidate
68

Samuel GAIST's avatar
Samuel GAIST committed
69
    return os.path.join(prefix, what, path + suffix)
70
71


Samuel GAIST's avatar
Samuel GAIST committed
72
# ----------------------------------------------------------
Philip ABBET's avatar
Philip ABBET committed
73

74

Philip ABBET's avatar
Philip ABBET committed
75
def safe_rmfile(f):
Philip ABBET's avatar
Philip ABBET committed
76
    """Safely removes a file from the disk"""
Philip ABBET's avatar
Philip ABBET committed
77

78
79
    if os.path.exists(f):
        os.unlink(f)
Philip ABBET's avatar
Philip ABBET committed
80
81


Samuel GAIST's avatar
Samuel GAIST committed
82
# ----------------------------------------------------------
Philip ABBET's avatar
Philip ABBET committed
83

Philip ABBET's avatar
Philip ABBET committed
84
85

def safe_rmdir(f):
Philip ABBET's avatar
Philip ABBET committed
86
    """Safely removes the directory containg a given file from the disk"""
Philip ABBET's avatar
Philip ABBET committed
87

Philip ABBET's avatar
Philip ABBET committed
88
    d = os.path.dirname(f)
89
90
91
92
93
94

    if not os.path.exists(d):
        return

    if not os.listdir(d):
        os.rmdir(d)
Philip ABBET's avatar
Philip ABBET committed
95
96


Samuel GAIST's avatar
Samuel GAIST committed
97
# ----------------------------------------------------------
Philip ABBET's avatar
Philip ABBET committed
98

Philip ABBET's avatar
Philip ABBET committed
99
100

def extension_for_language(language):
Philip ABBET's avatar
Philip ABBET committed
101
    """Returns the preferred extension for a given programming language
Philip ABBET's avatar
Philip ABBET committed
102

Philip ABBET's avatar
Philip ABBET committed
103
104
    The set of languages supported must match those declared in our
    ``common.json`` schema.
Philip ABBET's avatar
Philip ABBET committed
105

Philip ABBET's avatar
Philip ABBET committed
106
    Parameters:
Philip ABBET's avatar
Philip ABBET committed
107

Philip ABBET's avatar
Philip ABBET committed
108
      language (str) The language for which you'd like to get the extension for.
Philip ABBET's avatar
Philip ABBET committed
109
110


Philip ABBET's avatar
Philip ABBET committed
111
    Returns:
Philip ABBET's avatar
Philip ABBET committed
112

Philip ABBET's avatar
Philip ABBET committed
113
      str: The extension for the given language, including a leading ``.`` (dot)
Philip ABBET's avatar
Philip ABBET committed
114
115


Philip ABBET's avatar
Philip ABBET committed
116
    Raises:
Philip ABBET's avatar
Philip ABBET committed
117

Philip ABBET's avatar
Philip ABBET committed
118
      KeyError: If the language is not defined in our internal dictionary.
Philip ABBET's avatar
Philip ABBET committed
119

Philip ABBET's avatar
Philip ABBET committed
120
    """
Philip ABBET's avatar
Philip ABBET committed
121

Samuel GAIST's avatar
Samuel GAIST committed
122
    return dict(unknown="", cxx=".so", matlab=".m", python=".py", r=".r")[language]
123
124


Samuel GAIST's avatar
Samuel GAIST committed
125
# ----------------------------------------------------------
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146


class Prefix(object):
    def __init__(self, paths=None):
        if isinstance(paths, list):
            self.paths = paths
        elif paths is not None:
            self.paths = [paths]
        else:
            self.paths = []

    def add(self, path):
        self.paths.append(path)

    def path(self, filename):
        for p in self.paths:
            fullpath = os.path.join(p, filename)
            if os.path.exists(fullpath):
                return fullpath

        return os.path.join(self.paths[0], filename)
Philip ABBET's avatar
Philip ABBET committed
147
148


Samuel GAIST's avatar
Samuel GAIST committed
149
# ----------------------------------------------------------
Philip ABBET's avatar
Philip ABBET committed
150

Philip ABBET's avatar
Philip ABBET committed
151
152

class File(object):
Philip ABBET's avatar
Philip ABBET committed
153
    """User helper to read and write file objects"""
Philip ABBET's avatar
Philip ABBET committed
154

Philip ABBET's avatar
Philip ABBET committed
155
    def __init__(self, path, binary=False):
Philip ABBET's avatar
Philip ABBET committed
156

Philip ABBET's avatar
Philip ABBET committed
157
158
        self.path = path
        self.binary = binary
Philip ABBET's avatar
Philip ABBET committed
159

Philip ABBET's avatar
Philip ABBET committed
160
    def exists(self):
Philip ABBET's avatar
Philip ABBET committed
161

Philip ABBET's avatar
Philip ABBET committed
162
        return os.path.exists(self.path)
Philip ABBET's avatar
Philip ABBET committed
163

Philip ABBET's avatar
Philip ABBET committed
164
    def load(self):
Philip ABBET's avatar
Philip ABBET committed
165

Samuel GAIST's avatar
Samuel GAIST committed
166
        mode = "rb" if self.binary else "rt"
167
168
        with open(self.path, mode) as f:
            return f.read()
Philip ABBET's avatar
Philip ABBET committed
169

Philip ABBET's avatar
Philip ABBET committed
170
    def try_load(self):
Philip ABBET's avatar
Philip ABBET committed
171

Philip ABBET's avatar
Philip ABBET committed
172
173
174
        if os.path.exists(self.path):
            return self.load()
        return None
Philip ABBET's avatar
Philip ABBET committed
175

Philip ABBET's avatar
Philip ABBET committed
176
    def backup(self):
Philip ABBET's avatar
Philip ABBET committed
177

178
        if not os.path.exists(self.path):
Samuel GAIST's avatar
Samuel GAIST committed
179
            return  # no point in backing-up
180

Samuel GAIST's avatar
Samuel GAIST committed
181
        backup = self.path + "~"
182
183
184
        if os.path.exists(backup):
            os.remove(backup)

Philip ABBET's avatar
Philip ABBET committed
185
        shutil.copy(self.path, backup)
Philip ABBET's avatar
Philip ABBET committed
186

Philip ABBET's avatar
Philip ABBET committed
187
    def save(self, contents):
Philip ABBET's avatar
Philip ABBET committed
188

Philip ABBET's avatar
Philip ABBET committed
189
        d = os.path.dirname(self.path)
190
191
        if not os.path.exists(d):
            os.makedirs(d)
Philip ABBET's avatar
Philip ABBET committed
192

193
194
        if os.path.exists(self.path):
            self.backup()
Philip ABBET's avatar
Philip ABBET committed
195

Samuel GAIST's avatar
Samuel GAIST committed
196
        mode = "wb" if self.binary else "wt"
197
        if self.binary:
Samuel GAIST's avatar
Samuel GAIST committed
198
            mode = "wb"
199
            if isinstance(contents, six.string_types):
Samuel GAIST's avatar
Samuel GAIST committed
200
                contents = contents.encode("utf-8")
201
        else:
Samuel GAIST's avatar
Samuel GAIST committed
202
            mode = "wt"
203
            if not isinstance(contents, six.string_types):
Samuel GAIST's avatar
Samuel GAIST committed
204
                contents = contents.decode("utf-8")
205

206
207
        with open(self.path, mode) as f:
            f.write(contents)
Philip ABBET's avatar
Philip ABBET committed
208

Philip ABBET's avatar
Philip ABBET committed
209
    def remove(self):
Philip ABBET's avatar
Philip ABBET committed
210

Philip ABBET's avatar
Philip ABBET committed
211
        safe_rmfile(self.path)
Samuel GAIST's avatar
Samuel GAIST committed
212
213
        safe_rmfile(self.path + "~")  # backup
        safe_rmdir(self.path)  # remove containing directory
Philip ABBET's avatar
Philip ABBET committed
214
215


Samuel GAIST's avatar
Samuel GAIST committed
216
# ----------------------------------------------------------
Philip ABBET's avatar
Philip ABBET committed
217

Philip ABBET's avatar
Philip ABBET committed
218

219
class AbstractStorage(object):
Philip ABBET's avatar
Philip ABBET committed
220

221
222
223
    asset_type = None
    asset_folder = None

Philip ABBET's avatar
Philip ABBET committed
224
    def __init__(self, path):
Philip ABBET's avatar
Philip ABBET committed
225

226
227
228
229
230
        if not all(
            [type(attr) == str for attr in [self.asset_type, self.asset_folder]]
        ):
            raise TypeError("asset_type and asset_folder must be configure properly")

Philip ABBET's avatar
Philip ABBET committed
231
        self.path = path
Samuel GAIST's avatar
Samuel GAIST committed
232
233
        self.json = File(self.path + ".json")
        self.doc = File(self.path + ".rst")
Philip ABBET's avatar
Philip ABBET committed
234

Philip ABBET's avatar
Philip ABBET committed
235
236
    def exists(self):
        """If the database declaration file exists"""
237

Philip ABBET's avatar
Philip ABBET committed
238
        return self.json.exists()
Philip ABBET's avatar
Philip ABBET committed
239

240
241
242
243
244
245
246
247
248
249
250
    def remove(self):
        """Removes the object from the disk"""

        self.json.remove()
        self.doc.remove()

    def hash(self):
        """The 64-character hash of the database declaration JSON"""

        raise NotImplementedError

Philip ABBET's avatar
Philip ABBET committed
251
252
    def load(self):
        """Loads the JSON declaration as a file"""
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275

        raise NotImplementedError

    def save(self):
        """Saves the JSON declaration as files"""

        raise NotImplementedError


class Storage(AbstractStorage):
    """Resolves paths for objects that provide only a description"""

    def __init__(self, path):
        super(Storage, self).__init__(path)

    def hash(self, description="description"):
        """Re-imp"""

        return hash.hashJSONFile(self.json.path, description)

    def load(self):
        """Re-imp"""

Samuel GAIST's avatar
Samuel GAIST committed
276
        tp = collections.namedtuple("Storage", ["declaration", "description"])
Philip ABBET's avatar
Philip ABBET committed
277
        return tp(self.json.load(), self.doc.try_load())
Philip ABBET's avatar
Philip ABBET committed
278

Philip ABBET's avatar
Philip ABBET committed
279
    def save(self, declaration, description=None):
280
281
        """Re-imp"""

Samuel GAIST's avatar
Samuel GAIST committed
282
283
        if description:
            self.doc.save(description.encode("utf8"))
Philip ABBET's avatar
Philip ABBET committed
284
285
286
        if not isinstance(declaration, six.string_types):
            declaration = simplejson.dumps(declaration, indent=4)
        self.json.save(declaration)
Philip ABBET's avatar
Philip ABBET committed
287
288


Samuel GAIST's avatar
Samuel GAIST committed
289
# ----------------------------------------------------------
Philip ABBET's avatar
Philip ABBET committed
290

Philip ABBET's avatar
Philip ABBET committed
291

292
class CodeStorage(AbstractStorage):
Philip ABBET's avatar
Philip ABBET committed
293
294
295
296
297
298
299
300
301
    """Resolves paths for objects that provide a description and code

    Parameters:

      language (str): One of the valdid programming languages

    """

    def __init__(self, path, language=None):
302
        super(CodeStorage, self).__init__(path)
Philip ABBET's avatar
Philip ABBET committed
303
304

        self._language = language or self.__auto_discover_language()
Samuel GAIST's avatar
Samuel GAIST committed
305
306
307
        self.code = File(
            self.path + extension_for_language(self._language), binary=True
        )
Philip ABBET's avatar
Philip ABBET committed
308
309
310
311
312
313

    def __auto_discover_language(self, json=None):
        """Discovers and sets the language from its own JSON descriptor"""
        try:
            text = json or self.json.load()
            json = simplejson.loads(text)
Samuel GAIST's avatar
Samuel GAIST committed
314
            return json["language"]
Philip ABBET's avatar
Philip ABBET committed
315
        except IOError:
Samuel GAIST's avatar
Samuel GAIST committed
316
            return "unknown"
Philip ABBET's avatar
Philip ABBET committed
317
318
319
320
321
322
323
324

    @property
    def language(self):
        return self._language

    @language.setter
    def language(self, value):
        self._language = value
Samuel GAIST's avatar
Samuel GAIST committed
325
326
327
        self.code = File(
            self.path + extension_for_language(self._language), binary=True
        )
Philip ABBET's avatar
Philip ABBET committed
328
329

    def hash(self):
330
        """Re-imp"""
Philip ABBET's avatar
Philip ABBET committed
331
332

        if self.code.exists():
Samuel GAIST's avatar
Samuel GAIST committed
333
334
335
336
337
338
            return hash.hash(
                dict(
                    json=hash.hashJSONFile(self.json.path, "description"),
                    code=hash.hashFileContents(self.code.path),
                )
            )
Philip ABBET's avatar
Philip ABBET committed
339
        else:
Samuel GAIST's avatar
Samuel GAIST committed
340
341
342
            return hash.hash(
                dict(json=hash.hashJSONFile(self.json.path, "description"))
            )
Philip ABBET's avatar
Philip ABBET committed
343
344

    def exists(self):
345
346
347
        """Re-imp"""

        return super(CodeStorage, self).exists() and self.code.exists()
Philip ABBET's avatar
Philip ABBET committed
348
349

    def load(self):
350
351
        """Re-imp"""

Samuel GAIST's avatar
Samuel GAIST committed
352
353
354
        tp = collections.namedtuple(
            "CodeStorage", ["declaration", "code", "description"]
        )
Philip ABBET's avatar
Philip ABBET committed
355
356
357
        return tp(self.json.load(), self.code.try_load(), self.doc.try_load())

    def save(self, declaration, code=None, description=None):
358
359
        """Re-imp"""

Philip ABBET's avatar
Philip ABBET committed
360
        if description:
Samuel GAIST's avatar
Samuel GAIST committed
361
            self.doc.save(description.encode("utf8"))
Philip ABBET's avatar
Philip ABBET committed
362
363
364
365
366
367

        if not isinstance(declaration, six.string_types):
            declaration = simplejson.dumps(declaration, indent=4)
        self.json.save(declaration)

        if code:
Samuel GAIST's avatar
Samuel GAIST committed
368
            if self._language == "unknown":
Philip ABBET's avatar
Philip ABBET committed
369
370
371
372
                self.language = self.__auto_discover_language(declaration)
            self.code.save(code)

    def remove(self):
373
374
375
        """Re-imp"""

        super(CodeStorage, self).remove()
Philip ABBET's avatar
Philip ABBET committed
376
        self.code.remove()
377
378


Samuel GAIST's avatar
Samuel GAIST committed
379
# ----------------------------------------------------------
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394


class NumpyJSONEncoder(simplejson.JSONEncoder):
    """Encodes numpy arrays and scalars

    See Also:

      :py:class:`simplejson.JSONEncoder`

    """

    def default(self, obj):
        if isinstance(obj, numpy.ndarray) or isinstance(obj, numpy.generic):
            return obj.tolist()
        elif isinstance(obj, numpy.dtype):
Samuel GAIST's avatar
Samuel GAIST committed
395
396
            if obj.name == "str":
                return "string"
397
398
            return obj.name
        return simplejson.JSONEncoder.default(self, obj)
399
400
401
402
403
404
405
406


# ----------------------------------------------------------


def has_argument(method, argument):
    try:
        from inspect import signature
Samuel GAIST's avatar
Samuel GAIST committed
407

408
409
410
411
        sig = signature(method)
        params = sig.parameters
    except ImportError:
        from inspect import getargspec
Samuel GAIST's avatar
Samuel GAIST committed
412

413
414
415
        params = getargspec(method).args

    return argument in params