common.py 51.7 KB
Newer Older
André Anjos's avatar
André Anjos committed
1
2
3
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

Samuel GAIST's avatar
Samuel GAIST committed
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
###################################################################################
#                                                                                 #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/               #
# Contact: beat.support@idiap.ch                                                  #
#                                                                                 #
# Redistribution and use in source and binary forms, with or without              #
# modification, are permitted provided that the following conditions are met:     #
#                                                                                 #
# 1. Redistributions of source code must retain the above copyright notice, this  #
# list of conditions and the following disclaimer.                                #
#                                                                                 #
# 2. Redistributions in binary form must reproduce the above copyright notice,    #
# this list of conditions and the following disclaimer in the documentation       #
# and/or other materials provided with the distribution.                          #
#                                                                                 #
# 3. Neither the name of the copyright holder nor the names of its contributors   #
# may be used to endorse or promote products derived from this software without   #
# specific prior written permission.                                              #
#                                                                                 #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED   #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE          #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE    #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL      #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR      #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER      #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,   #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE   #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.            #
#                                                                                 #
###################################################################################
André Anjos's avatar
André Anjos committed
35
36


37
"""Utility functions that are useful to all sub-commands"""
André Anjos's avatar
André Anjos committed
38
39
40
41
42
43
44
45
46
47
48
49
50

import os
import glob
import fnmatch
import difflib
import collections

import logging

import six
import termcolor
import simplejson

51
52
from enum import Enum, unique

André Anjos's avatar
André Anjos committed
53
54
55
56
from beat.core import dataformat
from beat.core import database
from beat.core import library
from beat.core import plotter
57
from beat.core import plotterparameter
58
from beat.core import protocoltemplate
André Anjos's avatar
André Anjos committed
59
60
61
62
from beat.core import algorithm
from beat.core import toolchain
from beat.core import experiment

63
64
logger = logging.getLogger(__name__)

André Anjos's avatar
André Anjos committed
65
TYPE_GLOB = {
66
67
68
69
70
71
72
73
74
    "dataformat": os.path.join("*", "*", "*.json"),
    "database": os.path.join("*", "*.json"),
    "library": os.path.join("*", "*", "*.json"),
    "algorithm": os.path.join("*", "*", "*.json"),
    "plotter": os.path.join("*", "*", "*.json"),
    "plotterparameter": os.path.join("*", "*", "*.json"),
    "protocoltemplate": os.path.join("*", "*.json"),
    "toolchain": os.path.join("*", "*", "*.json"),
    "experiment": os.path.join("*", "*", "*", "*", "*.json"),
Philip ABBET's avatar
Philip ABBET committed
75
}
André Anjos's avatar
André Anjos committed
76
77
78


TYPE_FNMATCH = {
79
80
81
82
83
84
85
86
87
    "dataformat": os.path.splitext(TYPE_GLOB["dataformat"])[0],
    "database": os.path.splitext(TYPE_GLOB["database"])[0],
    "library": os.path.splitext(TYPE_GLOB["library"])[0],
    "algorithm": os.path.splitext(TYPE_GLOB["algorithm"])[0],
    "plotter": os.path.splitext(TYPE_GLOB["plotter"])[0],
    "plotterparameter": os.path.splitext(TYPE_GLOB["plotterparameter"])[0],
    "protocoltemplate": os.path.splitext(TYPE_GLOB["protocoltemplate"])[0],
    "toolchain": os.path.splitext(TYPE_GLOB["toolchain"])[0],
    "experiment": os.path.splitext(TYPE_GLOB["experiment"])[0],
Philip ABBET's avatar
Philip ABBET committed
88
}
André Anjos's avatar
André Anjos committed
89
90
91


TYPE_VALIDATOR = {
92
93
94
95
96
    "dataformat": dataformat.DataFormat,
    "database": database.Database,
    "library": library.Library,
    "algorithm": algorithm.Algorithm,
    "plotter": plotter.Plotter,
97
    "plotterparameter": plotterparameter.Plotterparameter,
98
99
100
    "protocoltemplate": protocoltemplate.ProtocolTemplate,
    "toolchain": toolchain.Toolchain,
    "experiment": experiment.Experiment,
Philip ABBET's avatar
Philip ABBET committed
101
}
André Anjos's avatar
André Anjos committed
102
103

TYPE_STORAGE = {
104
105
106
107
108
    "dataformat": dataformat.Storage,
    "database": database.Storage,
    "library": library.Storage,
    "algorithm": algorithm.Storage,
    "plotter": plotter.Storage,
109
    "plotterparameter": plotterparameter.Storage,
110
111
112
    "protocoltemplate": protocoltemplate.Storage,
    "toolchain": toolchain.Storage,
    "experiment": experiment.Storage,
Philip ABBET's avatar
Philip ABBET committed
113
}
André Anjos's avatar
André Anjos committed
114
115

TYPE_PLURAL = {
116
117
118
119
120
121
122
123
124
125
    "dataformat": "dataformats",
    "database": "databases",
    "library": "libraries",
    "algorithm": "algorithms",
    "plotter": "plotters",
    "plotterparameter": "plotters/plotterparameters",
    "defaultplotter": "plotters/defaultplotters",
    "toolchain": "toolchains",
    "experiment": "experiments",
    "protocoltemplate": "protocoltemplates",
Philip ABBET's avatar
Philip ABBET committed
126
}
André Anjos's avatar
André Anjos committed
127
128


129
130
131
132
133
134
135
136
137
138
139
140
141
@unique
class ModificationStatus(Enum):
    """This enum describes the state of possible changes between a local asset
    and it's remote counter part"""

    NO_CHANGES = ""
    REMOTE_ONLY_AVAILABLE = "r"
    LOCAL_ONLY_AVAILABLE = "l"
    DOC_CHANGED = "d"
    CONTENT_CHANGED = "+"
    BOTH_CHANGED = "*"


André Anjos's avatar
André Anjos committed
142
def recursive_rmdir_if_empty(path, stop_at):
143
144
145
146
147
148
149
150
151
152
153
    """Recursively removes empty directories until a certain top directory"""

    if not os.path.exists(path):
        recursive_rmdir_if_empty(os.path.dirname(path), stop_at)
        return
    if os.path.samefile(path, stop_at):
        return  # stop
    if not os.listdir(path):  # empty
        logger.info("removing empty directory `%s'...", path)
        os.rmdir(path)
        recursive_rmdir_if_empty(os.path.dirname(path), stop_at)
André Anjos's avatar
André Anjos committed
154
155
156
157
    return


class Selector(object):
158
159
160
161
162
163
164
    """Keeps track of versions and fork status"""

    def __init__(self, prefix):

        self.prefix = prefix  # the root of the directory
        self.path = os.path.join(self.prefix, ".beat", "selected.json")

165
166
167
        self.__version = {}
        self.__fork = {}
        self.__versionables = [
168
            "algorithm",
169
170
171
172
            "dataformat",
            "database",
            "library",
            "toolchain",
173
174
            "plotter",
            "plotterparameter",
175
176
177
            "protocoltemplate",
        ]

178
179
180
181
182
183
184
185
186
        self.__forkables = [
            "algorithm",
            "dataformat",
            "experiment",
            "library",
            "toolchain",
            "plotter",
            "plotterparameter",
        ]
187

188
        if os.path.exists(self.path):
189
            self.load()
190
        else:
191
            self.__ensure_entries()
192
193
194
195

    def __enter__(self):
        """Implements our context manager"""
        return self
André Anjos's avatar
André Anjos committed
196

197
198
199
    def __exit__(self, *exc):
        """Implements our context manager"""
        self.save()
André Anjos's avatar
André Anjos committed
200

201
202
203
    def __ensure_entries(self):
        """Ensure all types have an entry"""

204
205
206
        for asset_type in self.__versionables:
            if asset_type not in self.__version:
                self.__version[asset_type] = dict()
207

208
209
210
        for asset_type in self.__forkables:
            if asset_type not in self.__fork:
                self.__fork[asset_type] = dict()
211

212
213
214
215
216
    def can_fork(self, asset_type):
        """Returns whether the given asset type can be forked"""

        return asset_type in self.__forkables

217
218
219
220
221
    def has_versions(self, asset_type):
        """Returns whether the given asset type can have versions"""

        return asset_type in self.__versionables

222
    def fork(self, asset_type, src, dst):
223
        """Registers that object ``dst`` is a fork of object ``src``"""
André Anjos's avatar
André Anjos committed
224

225
226
227
        if not self.can_fork(asset_type):
            raise RuntimeError("Can't create new version of {}".format(asset_type))

228
229
        logger.info(
            "`%s/%s' is forked from `%s/%s'",
230
            TYPE_PLURAL[asset_type],
231
            dst,
232
            TYPE_PLURAL[asset_type],
233
234
            src,
        )
235
        self.__fork[asset_type][dst] = src
André Anjos's avatar
André Anjos committed
236

237
    def forked_from(self, asset_type, name):
238
        """Returns the name of the originating source object or ``None``"""
239
240
        if not self.can_fork(asset_type):
            return None
André Anjos's avatar
André Anjos committed
241

242
        return self.__fork[asset_type].get(name)
André Anjos's avatar
André Anjos committed
243

244
    def version(self, asset_type, src, dst):
245
        """Registers that object ``dst`` is a new version of object ``src``"""
André Anjos's avatar
André Anjos committed
246

247
        if asset_type not in self.__versionables:
248
            raise RuntimeError("Can't create new version of {}".format(asset_type))
249

250
251
        logger.info(
            "`%s/%s' is a new version of `%s/%s'",
252
            TYPE_PLURAL[asset_type],
253
            dst,
254
            TYPE_PLURAL[asset_type],
255
256
            src,
        )
257
        self.__version[asset_type][dst] = src
André Anjos's avatar
André Anjos committed
258

259
    def version_of(self, asset_type, name):
260
        """Returns the name of the originating version object or ``None``"""
André Anjos's avatar
André Anjos committed
261

262
        if asset_type not in self.__version:
263
264
            return None

265
        return self.__version[asset_type].get(name)
André Anjos's avatar
André Anjos committed
266

267
    def delete(self, asset_type, name):
268
        """Forgets about an object that was being tracked"""
André Anjos's avatar
André Anjos committed
269

270
        if asset_type in self.__fork and name in self.__fork[asset_type]:
271
272
273
            del self.__fork[asset_type][name]
        if asset_type in self.__version and name in self.__version[asset_type]:
            del self.__version[asset_type][name]
André Anjos's avatar
André Anjos committed
274

275
276
    def load(self):
        """Loads contents from file"""
André Anjos's avatar
André Anjos committed
277

278
279
280
281
        try:
            with open(self.path, "rt") as f:
                data = simplejson.load(f, object_pairs_hook=collections.OrderedDict)
        except simplejson.JSONDecodeError:
282
            logger.warning(
283
284
285
                "invalid state file at `%s' - removing and re-starting...", self.path
            )
            from beat.core.utils import safe_rmfile
André Anjos's avatar
André Anjos committed
286

287
288
            safe_rmfile(self.path)
            return False
André Anjos's avatar
André Anjos committed
289

290
291
        self.__fork = data["fork"]
        self.__version = data["version"]
292
293
        self.__ensure_entries()

294
        return True
André Anjos's avatar
André Anjos committed
295

296
297
    def save(self):
        """Saves contents to file"""
André Anjos's avatar
André Anjos committed
298

299
300
301
302
303
304
        dirname = os.path.dirname(self.path)
        if not os.path.exists(dirname):
            os.makedirs(dirname)
        data = {"fork": self.__fork, "version": self.__version}
        with open(self.path, "wt") as f:
            simplejson.dump(data, f, indent=2)
André Anjos's avatar
André Anjos committed
305
306


307
def retrieve_remote_list(webapi, asset_type, fields):
308
    """Utility function used by commands to retrieve a remote list of objects
André Anjos's avatar
André Anjos committed
309
310
311
312
313
314
315


  Parameters:

    webapi (object): An instance of our WebAPI class, prepared to access the
      BEAT server of interest

316
    asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
André Anjos's avatar
André Anjos committed
317
318
      ``toolchain`` or ``experiment``.

André Anjos's avatar
André Anjos committed
319
320
    fields (:py:class:`list`): A list of fields to retrieve from the remote
      server
André Anjos's avatar
André Anjos committed
321
322
323
324


  Returns:

André Anjos's avatar
André Anjos committed
325
326
    :py:class:`list`: A list of dictionaries containing the ``name``,
    ``short_description`` and ``hash`` of available remote objects.
André Anjos's avatar
André Anjos committed
327

328
  """
André Anjos's avatar
André Anjos committed
329

330
    logger.debug("retrieving remote %s list...", TYPE_PLURAL[asset_type])
André Anjos's avatar
André Anjos committed
331

332
    fields = "" if not fields else "?fields=%s" % ",".join(fields)
André Anjos's avatar
André Anjos committed
333

334
    url = "/api/v1/%s/%s" % (TYPE_PLURAL[asset_type], fields)
André Anjos's avatar
André Anjos committed
335

336
    return webapi.get(url)
André Anjos's avatar
André Anjos committed
337
338


339
def make_up_remote_list(webapi, asset_type, requirements):
340
    """Creates a list of downloadable objects from user requirements.
André Anjos's avatar
André Anjos committed
341
342
343
344
345
346
347
348
349
350
351
352
353

  This function can create a list of downloadable objects from user
  requirements. User requirements may point to valid object names (in which
  case these are returned unchanged) or partial object names, which are used to
  filter available remote resources. A list of fully resolved remote names
  respecting user restrictions is returned.


  Parameters:

    webapi (object): An instance of our WebAPI class, prepared to access the
      BEAT server of interest

354
    asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
André Anjos's avatar
André Anjos committed
355
356
      ``toolchain`` or ``experiment``.

André Anjos's avatar
André Anjos committed
357
358
    requirements (:py:class:`list`): A list of requirements that are used to
      filter (additively) the available (remote) objects.
André Anjos's avatar
André Anjos committed
359
360
361
362


  Returns:

André Anjos's avatar
André Anjos committed
363
364
    :py:class:`list`: A list of valid object names matching user requirements
      and its order.
André Anjos's avatar
André Anjos committed
365

366
  """
André Anjos's avatar
André Anjos committed
367

368
    candidates = retrieve_remote_list(webapi, asset_type, ["name"])
André Anjos's avatar
André Anjos committed
369

370
371
372
373
    if not requirements:  # special case, return all possible values
        if candidates is None:
            return None
        return [c["name"] for c in candidates]
André Anjos's avatar
André Anjos committed
374

375
    # othewise, we need to separate filters from full-names
376
    full_requirements = fnmatch.filter(requirements, TYPE_FNMATCH[asset_type])
377
    short_requirements = [k for k in requirements if k not in full_requirements]
André Anjos's avatar
André Anjos committed
378

379
    retval = []
André Anjos's avatar
André Anjos committed
380

381
382
383
    if short_requirements:
        if candidates is None:
            return None
384
        retval = set()
385
        for name in short_requirements:
386
            retval |= set([k["name"] for k in candidates if k["name"].find(name) != -1])
387
388
        retval = list(retval)
        logger.info("search strings matched %d remote object(s)", len(retval))
André Anjos's avatar
André Anjos committed
389

390
391
392
    # note: if you specify a full-length requirement, we don't really care if it
    # is there or not. The final command will decide if it is an error.
    return retval + full_requirements
André Anjos's avatar
André Anjos committed
393
394


395
def display_remote_list(webapi, asset_type):
396
    """Implements a generic "list --remote" command
André Anjos's avatar
André Anjos committed
397
398
399
400
401
402

  Parameters:

    webapi (object): An instance of our WebAPI class, prepared to access the
      BEAT server of interest, on behalf of a pre-configured user.

403
    asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
André Anjos's avatar
André Anjos committed
404
405
406
407
408
409
410
411
412
      ``toolchain`` or ``experiment``.


  Returns:

    int: Indicating the exit status of the command, to be reported back to the
      calling process. This value should be zero if everything works OK,
      otherwise, different than zero (POSIX compliance).

413
  """
André Anjos's avatar
André Anjos committed
414

415
416
417
    remote_list = retrieve_remote_list(
        webapi, asset_type, ["name", "short_description"]
    )
418
419
    if remote_list is None:
        return 1
André Anjos's avatar
André Anjos committed
420

421
422
423
424
    for item in remote_list:
        logger.info("%s", item["name"])
        if item["short_description"]:
            logger.extra(2 * " " + item["short_description"])
André Anjos's avatar
André Anjos committed
425

426
    if len(remote_list) != 1:
427
        logger.extra("%d %s found", len(remote_list), TYPE_PLURAL[asset_type])
428
    else:
429
        logger.extra("1 %s found" % asset_type)
André Anjos's avatar
André Anjos committed
430

431
    return 0
André Anjos's avatar
André Anjos committed
432
433


434
def make_up_local_list(prefix, asset_type, requirements):
435
    """Creates a list of uploadable objects from user requirements.
André Anjos's avatar
André Anjos committed
436
437
438
439
440
441
442
443
444
445
446
447
448

  This function can create a list of uploadable objects from user requirements.
  User requirements may point to valid object names (in which case these are
  returned unchanged) or partial object names, which are used to filter
  available local resources. A list of fully resolved local names respecting
  user restrictions is returned.


  Parameters:

    prefix (str): A string representing the root of the path in which the user
      objects are stored

449
    asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
André Anjos's avatar
André Anjos committed
450
451
      ``toolchain`` or ``experiment``.

André Anjos's avatar
André Anjos committed
452
453
    requirements (:py:class:`list`): A list of requirements that are used to
      filter (additively) the available (remote) objects.
André Anjos's avatar
André Anjos committed
454
455
456
457


  Returns:

André Anjos's avatar
André Anjos committed
458
459
460
    :py:class:`list`: A list of strings, each with the relative name of an
      object belonging to a certain category and in the order prescribed by the
      user.
André Anjos's avatar
André Anjos committed
461

462
  """
André Anjos's avatar
André Anjos committed
463

464
465
    root = os.path.join(prefix, TYPE_PLURAL[asset_type])
    asset_path_list = glob.glob(os.path.join(root, TYPE_GLOB[asset_type]))
466
467
468
469
470
    candidates = [
        os.path.splitext(os.path.relpath(path, root))[0] for path in asset_path_list
    ]

    # adds hashed path structures
471
    hashed_path_list = glob.glob(os.path.join(root, "*", "*", TYPE_GLOB[asset_type]))
472
473
474
475
476
477
478
479
480
481
    hashed_path_list = [
        os.path.splitext(os.path.relpath(path, root))[0] for path in hashed_path_list
    ]
    candidates += [os.path.join(*path.split(os.sep)[2:]) for path in hashed_path_list]

    if not requirements:
        return candidates

    use_requirements = []
    for k in requirements:  # remove leading plural-name
482
483
        if k.startswith(TYPE_PLURAL[asset_type] + os.sep):
            use_requirements.append(k.replace(TYPE_PLURAL[asset_type] + os.sep, ""))
484
485
486
        else:
            use_requirements.append(k)
    requirements = use_requirements
André Anjos's avatar
André Anjos committed
487

488
    full_requirements = fnmatch.filter(requirements, TYPE_FNMATCH[asset_type])
489
    short_requirements = [k for k in requirements if k not in full_requirements]
André Anjos's avatar
André Anjos committed
490

491
    retval = set()
492
    for name in short_requirements:
493
        retval |= set([k for k in candidates if k.startswith(name)])
André Anjos's avatar
André Anjos committed
494

495
496
497
    # note: if you specify a full-length requirement, we don't really care if it
    # is there or not. The final command will decide if it is an error.
    return list(retval) + full_requirements
André Anjos's avatar
André Anjos committed
498
499


500
def display_local_list(prefix, asset_type):
501
    """Implements the local "list" command
André Anjos's avatar
André Anjos committed
502
503
504
505
506
507
508


  Parameters:

    prefix (str): A string representing the root of the path in which the user
      objects are stored

509
    asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
André Anjos's avatar
André Anjos committed
510
511
512
513
514
515
516
517
518
      ``toolchain`` or ``experiment``.


  Returns:

    int: Indicating the exit status of the command, to be reported back to the
      calling process. This value should be zero if everything works OK,
      otherwise, different than zero (POSIX compliance).

519
  """
André Anjos's avatar
André Anjos committed
520

521
    names = make_up_local_list(prefix, asset_type, [])
522
523
524
525

    for name in names:
        logger.info("%s", name)
        try:
526
527
528
529
            storage = TYPE_STORAGE[asset_type](prefix, name)
            contents = simplejson.loads(
                storage.json.load(), object_pairs_hook=collections.OrderedDict
            )
530
531
532
            if "description" in contents:
                logger.extra(2 * " " + contents["description"])
        except simplejson.JSONDecodeError:
533
            logger.warning(2 * " " + "(!) invalid JSON file")
534
535

    if len(names) != 1:
536
        logger.extra("%d %s found", len(names), TYPE_PLURAL[asset_type])
537
    else:
538
        logger.extra("1 %s found" % asset_type)
André Anjos's avatar
André Anjos committed
539

540
    return 0
André Anjos's avatar
André Anjos committed
541
542


543
def display_local_path(prefix, asset_type, names):
544
    """Implements the local "path" command
545
546
547
548
549
550
551


  Parameters:

    prefix (str): A string representing the root of the path in which the user
      objects are stored

552
    asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
553
554
555
556
557
558
559
560
561
      ``toolchain`` or ``experiment``.


  Returns:

    int: Indicating the exit status of the command, to be reported back to the
      calling process. This value should be zero if everything works OK,
      otherwise, different than zero (POSIX compliance).

562
  """
563

564
    selected_type = None
565

566
567
568
569
570
    try:
        selected_type = TYPE_PLURAL[asset_type]
    except IndexError:
        logger.error("Selected type is not valid: %s", asset_type)
        return 1
571

572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
    for name in names:
        root = os.path.join(prefix, selected_type)
        object_path = os.path.join(root, name.rsplit("/", 1)[0])
        object_files = [
            filename
            for filename in os.listdir(object_path)
            if filename.startswith(name.rsplit("/", 1)[1])
        ]
        if len(object_files) > 0:
            logger.info(
                "Available local file(s) for type '%s' and name '%s':",
                selected_type,
                name,
            )
            for filename in object_files:
                full_name = os.path.join(object_path, filename)
                logger.info(full_name)
        else:
            logger.info(
                "No local file(s) found for type '%s' and name '%s':",
                selected_type,
                name,
            )
595

596
    return 0
597
598


599
def edit_local_file(prefix, editor, asset_type, name):
600
    """Implements the local "path" command
601
602
603
604
605
606
607


  Parameters:

    prefix (str): A string representing the root of the path in which the user
      objects are stored

608
    asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
609
610
611
612
613
614
615
616
617
      ``toolchain`` or ``experiment``.


  Returns:

    int: Indicating the exit status of the command, to be reported back to the
      calling process. This value should be zero if everything works OK,
      otherwise, different than zero (POSIX compliance).

618
  """
619

620
    selected_type = None
621

622
623
624
625
626
    try:
        selected_type = TYPE_PLURAL[asset_type]
    except IndexError:
        logger.error("Selected type is not valid: %s", asset_type)
        return 1
627

628
629
630
631
632
633
634
635
636
637
    python_objects = ["database", "library", "algorithm", "plotter"]
    json_objects = [
        "dataformat",
        "toolchain",
        "experiment",
        "plotterparameter",
        "protocoltemplate",
    ]

    ext = None
638
    if asset_type in python_objects:
639
        ext = ".py"
640
    elif asset_type in json_objects:
641
642
        ext = ".json"
    else:
643
        logger.error("Selected type is not valid: %s", asset_type)
644

645
646
    root = os.path.join(prefix, selected_type)
    object_path = os.path.join(root, name + ext)
647
648
    if os.path.isfile(object_path):
        # check if editor set
649
        if editor is None:
650
651
652
653
654
655
656
            if "VISUAL" in os.environ and len(os.environ["VISUAL"]) > 0:
                editor = os.environ["VISUAL"]
            elif "EDITOR" in os.environ and len(os.environ["EDITOR"]) > 0:
                editor = os.environ["EDITOR"]
            else:
                logger.error("No default editor set in your environment variable")
                return 1
657
        logger.info("Editing object of type '%s' and name '%s'", selected_type, name)
658
659
        cmd = "%s %s" % (editor, object_path)
        os.system(cmd)  # nosec
660
    else:
661
        logger.error("Not a valid file: %s", object_path)
662
663
        return 1

664
665
    return 0

666

667
def make_webapi(config):
668
    """Instantiates an usable web-api proxy using the command-line configuration
André Anjos's avatar
André Anjos committed
669
670
671

  Parameters:

672
    config (object): The command-line configuration object, from which this function
André Anjos's avatar
André Anjos committed
673
674
675
676
677
678
679
      will extract the ``platform``, ``user`` and ``token`` parameters.


  Returns

    WebAPI: A valid web-api proxy instance

680
  """
André Anjos's avatar
André Anjos committed
681

682
    from .webapi import WebAPI
André Anjos's avatar
André Anjos committed
683

684
    return WebAPI(config.platform, config.user, config.token)
André Anjos's avatar
André Anjos committed
685
686


687
def check_one(prefix, asset_type, name):
688
    """Implements object validation for a single, well-defined object
André Anjos's avatar
André Anjos committed
689
690
691
692
693
694
695


  Parameters:

    prefix (str): A string representing the root of the path in which the user
      objects are stored

696
    asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
André Anjos's avatar
André Anjos committed
697
698
699
700
701
702
703
704
705
706
      ``toolchain`` or ``experiment``.

    name (str): The name of the object, representing the unique relative path
      of the objects to check (e.g. ``user/integer/1``)

    klass (type): A python class that validates the object. It must accept the
      object

  """

707
    o = TYPE_VALIDATOR[asset_type](prefix, name)
André Anjos's avatar
André Anjos committed
708

709
    if not o.valid:
710
        logger.info("%s/%s [invalid]", TYPE_PLURAL[asset_type], name)
711
        for e in o.errors:
712
            logger.warning("  * %s", e)
713
        return 1
André Anjos's avatar
André Anjos committed
714

715
    else:
716
        logger.info("%s/%s [ok]", TYPE_PLURAL[asset_type], name)
717
        return 0
André Anjos's avatar
André Anjos committed
718
719


720
def check(prefix, asset_type, names):
721
    """Implements object validation
André Anjos's avatar
André Anjos committed
722
723
724
725
726
727
728


  Parameters:

    prefix (str): A string representing the root of the path in which the user
      objects are stored

729
    asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
André Anjos's avatar
André Anjos committed
730
731
      ``toolchain`` or ``experiment``.

André Anjos's avatar
André Anjos committed
732
    names (:py:class:`list`): A list of strings, each representing the unique
André Anjos's avatar
André Anjos committed
733
734
735
736
737
738
739
740
741
742
743
744
      relative path of the objects to check. If the list is empty, then we
      check all available objects of a given type.


  Returns:

    int: Indicating the exit status of the command, to be reported back to the
      calling process. This value should be zero if everything works OK,
      otherwise, different than zero (POSIX compliance).

  """

745
746
    names = make_up_local_list(prefix, asset_type, names)
    return sum([check_one(prefix, asset_type, name) for name in names])
André Anjos's avatar
André Anjos committed
747
748


749
def fetch_object(webapi, asset_type, name, fields):
750
    """Retrieves a single well-known object from the server
André Anjos's avatar
André Anjos committed
751
752
753
754
755
756

  Parameters:

    webapi (object): An instance of our WebAPI class, prepared to access the
      BEAT server of interest

757
    asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
André Anjos's avatar
André Anjos committed
758
759
760
761
      ``toolchain`` or ``experiment``.

    name (str): A string defining the name of the object to retrieve

André Anjos's avatar
André Anjos committed
762
763
    fields (:py:class:`list`): A list of fields to retrieve from the remote
      server
André Anjos's avatar
André Anjos committed
764
765
766
767


  Returns:

768
    dict: A dictionary containing the object contents
André Anjos's avatar
André Anjos committed
769
770
771

  """

772
773
    fields = "?object_format=string&fields=%s" % ",".join(fields)
    if name is not None:
774
        url = "/api/v1/%s/%s/%s" % (TYPE_PLURAL[asset_type], name, fields)
775
    else:
776
        url = "/api/v1/%s/%s" % (TYPE_PLURAL[asset_type], fields)
André Anjos's avatar
André Anjos committed
777

778
    return webapi.get(url)
André Anjos's avatar
André Anjos committed
779
780


781
def pull(webapi, prefix, asset_type, names, fields, force, indentation):
782
    """Copies objects from the server to the local prefix
André Anjos's avatar
André Anjos committed
783
784
785
786
787
788
789
790
791
792


  Parameters:

    webapi (object): An instance of our WebAPI class, prepared to access the
      BEAT server of interest

    prefix (str): A string representing the root of the path in which the user
      objects are stored

793
    asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
André Anjos's avatar
André Anjos committed
794
795
      ``toolchain`` or ``experiment``.

André Anjos's avatar
André Anjos committed
796
    names (:py:class:`list`): A list of strings, each representing the unique
André Anjos's avatar
André Anjos committed
797
798
799
800
801
      relative path of the objects to retrieve or a list of usernames from
      which to retrieve objects. If the list is empty, then we pull all
      available objects of a given type. If no user is set, then pull all
      public objects of a given type.

André Anjos's avatar
André Anjos committed
802
    fields (:py:class:`list`): A list of strings, each defining one field that
André Anjos's avatar
André Anjos committed
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
      **must** be downloaded from the web-server for a given object of the
      current type and passed, unchanged to the storage ``save()`` method. For
      example, for toolchains, this value shall be ``['declaration']``. For
      algorithms, it shall be ``['declaration', 'code']``.

    force (bool): If set to ``True``, then overwrites local changes with the
      remotely retrieved copies.

    indentation (int): The indentation level, useful if this function is called
      recursively while downloading different object types. This is normally
      set to ``0`` (zero).


  Returns:

    int: Indicating the exit status of the command, to be reported back to the
      calling process. This value should be zero if everything works OK,
      otherwise, different than zero (POSIX compliance).

André Anjos's avatar
André Anjos committed
822
823
824
825
    :py:class:`list`: A list of strings containing the names of objects
      successfuly downloaded or which were already present on the current
      installation (if the user has chosen not to ``--force`` the override), in
      the order of their download.
André Anjos's avatar
André Anjos committed
826
827
828

  """

829
    names = make_up_remote_list(webapi, asset_type, names)
830
831
832
    if not names:
        return 1, []
    indent = indentation * " "
833
    available = set()
834
835
836
837

    status = 0

    for name in names:
838
839
840
841
842
843
844
845
846
847
        storage = TYPE_STORAGE[asset_type](prefix, name)
        if storage.exists() and not force:  # exists locally, force not set
            logger.extra(
                "%sskipping download of `%s/%s' (exists locally)",
                indent,
                TYPE_PLURAL[asset_type],
                name,
            )
            available.add(name)
            continue
848
        else:
849
850
851
            logger.info(
                "%sretrieving `%s/%s'...", indent, TYPE_PLURAL[asset_type], name
            )
852

853
854
855
            data = fetch_object(webapi, asset_type, name, fields)
            if data is None:
                status += 1  # error
856
                continue
857
858
859
860
861
862
863
864

            if asset_type == "plotterparameter":
                declaration = {
                    "description": data["short_description"],
                    "plotter": data["plotter"],
                    "data": data["data"],
                }
                storage.save(declaration)
865
            else:
866
867
                storage.save(**data)
            available.add(name)
868
869

    return status, list(available)
André Anjos's avatar
André Anjos committed
870
871


872
def diff(webapi, prefix, asset_type, name, fields):
873
    """Shows the differences between two objects, for each of the fields
André Anjos's avatar
André Anjos committed
874
875
876
877
878
879
880
881
882
883


  Parameters:

    webapi (object): An instance of our WebAPI class, prepared to access the
      BEAT server of interest

    prefix (str): A string representing the root of the path in which the user
      objects are stored

884
    asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
André Anjos's avatar
André Anjos committed
885
886
887
888
889
      ``toolchain`` or ``experiment``.

    name (str): A string defining the name of the object to calculate
      differences from.

André Anjos's avatar
André Anjos committed
890
    fields (:py:class:`list`): A list of strings, each defining one field that
André Anjos's avatar
André Anjos committed
891
892
893
894
895
896
897
898
899
900
901
902
903
      **must** be downloaded from the web-server for a given object of the
      current type.  For example, for toolchains, this value shall be
      ``['declaration']``. For algorithms, it shall be ``['declaration',
      'code']``.

  Returns:

    int: Indicating the exit status of the command, to be reported back to the
      calling process. This value should be zero if everything works OK,
      otherwise, different than zero (POSIX compliance).

  """

904
    extension = {"code": ".py", "declaration": ".json", "description": ".rst"}
André Anjos's avatar
André Anjos committed
905

906
907
    def _eval_diff(remote, local, ext):
        """Calculates differences between two string buffers"""
André Anjos's avatar
André Anjos committed
908

909
        if not isinstance(local, six.string_types):
910
911
912
913
            if isinstance(local, dict):
                local = simplejson.dumps(local)
            else:
                local = local.decode("utf-8")
914
        if not isinstance(remote, six.string_types):
915
916
917
918
            if isinstance(remote, dict):
                remote = simplejson.dumps(remote)
            else:
                remote = remote.decode("utf-8")
André Anjos's avatar
André Anjos committed
919

920
921
922
        return difflib.unified_diff(
            remote.split("\n"),
            local.split("\n"),
923
924
            os.path.join("remote", asset_type, name + ext),
            os.path.join("local", asset_type, name + ext),
925
        )
André Anjos's avatar
André Anjos committed
926

927
928
    def _show_diff(diffs):
        """Displays difference display between two string buffers"""
André Anjos's avatar
André Anjos committed
929

930
931
932
933
934
935
936
        for line in diffs:
            if line.startswith("+"):
                termcolor.cprint(line, "green")
            elif line.startswith("-"):
                termcolor.cprint(line, "red")
            else:
                print(line)
André Anjos's avatar
André Anjos committed
937

938
    storage = TYPE_STORAGE[asset_type](prefix, name)
939
    local = storage.load()  # may also return a tuple, depending on the type
940
    remote = fetch_object(webapi, asset_type, name, fields)
941
942
943
944
945
946
947
948
949
950
951
952
953
954
    if remote is None:
        return 1
    if "declaration" in remote and not isinstance(
        remote["declaration"], six.string_types
    ):
        remote["declaration"] = simplejson.dumps(remote["declaration"], indent=4)

    local = dict(zip(fields, local))  # ``local`` should have the same size

    # replaces None entries with an empty string so these are comparable
    for key in local:
        local[key] = local[key] if local[key] is not None else ""

    for field in fields:
955
        diffs = _eval_diff(remote[field], local[field], extension.get(field, ""))
956
957
        if diffs:
            logger.info(
958
                "differences for `%s' of `%s/%s':", field, TYPE_PLURAL[asset_type], name
959
960
961
962
            )
            _show_diff(diffs)
        else:
            logger.info(
963
964
965
966
                "no differences for `%s' of `%s/%s'",
                field,
                TYPE_PLURAL[asset_type],
                name,
967
            )
André Anjos's avatar
André Anjos committed
968

969
    return 0
André Anjos's avatar
André Anjos committed
970
971


972
def create(prefix, asset_type, names):
973
    """Creates an empty object of a certain type under the given name
André Anjos's avatar
André Anjos committed
974
975
976
977
978
979
980


  Parameters:

    prefix (str): A string representing the root of the path in which the user
      objects are stored

981
    asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
André Anjos's avatar
André Anjos committed
982
983
984
985
986
987
988
989
990
991
992
993
994
      ``toolchain`` or ``experiment``.

    names (str): A string defining the names of the objects to create.


  Returns:

    int: Indicating the exit status of the command, to be reported back to the
      calling process. This value should be zero if everything works OK,
      otherwise, different than zero (POSIX compliance).

  """

995
    status = 0
André Anjos's avatar
André Anjos committed
996

997
    for name in names:
André Anjos's avatar
André Anjos committed
998

999
        storage = TYPE_STORAGE[asset_type](prefix, name)
André Anjos's avatar
André Anjos committed
1000

1001
1002
        if storage.exists():
            logger.error(
1003
1004
1005
                "`%s/%s' already exists - will *not* overwrite",
                TYPE_PLURAL[asset_type],
                name,
1006
1007
            )
            status += 1
André Anjos's avatar
André Anjos committed
1008

1009
1010
        obj = TYPE_VALIDATOR[asset_type](prefix, data=None)  # the default object
        storage = TYPE_STORAGE[asset_type](prefix, name)
1011
        obj.write(storage)
André Anjos's avatar
André Anjos committed
1012

1013
    return status
André Anjos's avatar
André Anjos committed
1014
1015


1016
def copy(prefix, asset_type, src, dst):
1017
    """Creates a new object by copying another object of the same type.
André Anjos's avatar
André Anjos committed
1018
1019
1020
1021
1022
1023

  Parameters:

    prefix (str): A string representing the root of the path in which the user
      objects are stored

1024
    asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
André Anjos's avatar
André Anjos committed
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
      ``toolchain`` or ``experiment``.

    src (str): A string defining the name of the object to fork a new version
      from.

    dst (str): A string defining the name of the object to fork to.


  Returns:

    int: Indicating the exit status of the command, to be reported back to the
      calling process. This value should be zero if everything works OK,
      otherwise, different than zero (POSIX compliance).

  """

1041
    src_storage = TYPE_STORAGE[asset_type](prefix, src)
1042
    if not src_storage.exists():
1043
        logger.error("source `%s/%s' does not exist", TYPE_PLURAL[asset_type], src)
1044
        return 1
André Anjos's avatar
André Anjos committed
1045

1046
    dst_storage = TYPE_STORAGE[asset_type](prefix, dst)
1047
    if dst_storage.exists():
1048
        logger.error("destination `%s/%s' already exists", TYPE_PLURAL[asset_type], dst)
1049
        return 1
André Anjos's avatar
André Anjos committed
1050

1051
    dst_storage.save(*src_storage.load())
André Anjos's avatar
André Anjos committed
1052

1053
    return 0
André Anjos's avatar
André Anjos committed
1054
1055


1056
def new_version(prefix, asset_type, src):
1057
    """Creates a new object by copying another object of the same type.
André Anjos's avatar
André Anjos committed
1058
1059
1060
1061
1062
1063

  Parameters:

    prefix (str): A string representing the root of the path in which the user
      objects are stored

1064
    asset_type (str): One of ``database``, ``dataformat``, ``algorithm``,
André Anjos's avatar
André Anjos committed
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
      ``toolchain`` or ``experiment``.

    src (str): A string defining the name of the object to fork a new version
      from.


  Returns:

    int: Indicating the exit status of the command, to be reported back to the
      calling process. This value should be zero if everything works OK,
      otherwise, different than zero (POSIX compliance).

  """

1079
    with Selector(prefix) as selector:
1080
        src_storage = TYPE_STORAGE[asset_type](prefix, src)
1081
1082
        dst = os.sep.join(src.split(os.sep)[:-1] + [""])
        dst += str(int(src_storage.version) + 1)
1083
        dst_storage = TYPE_STORAGE[asset_type](prefix, dst)
1084
1085
1086
1087

        if dst_storage.exists():
            logger.info(
                "A representation for %s `%s' already exists - not " "overwriting",
1088
                asset_type,
1089
1090
1091
                dst,
            )
        else:
1092
            status = copy(prefix, asset_type, src, dst)
1093
1094
            if status != 0:
                return status  # error
André Anjos's avatar
André Anjos committed
1095

1096
        selector.version(asset_type, src, dst)
1097
        return 0
André Anjos's avatar
André Anjos committed
1098
1099


1100
def fork(prefix, asset_type, src, dst):
1101
    """Creates a new object by forking another object of the same type.
André Anjos's avatar
André Anjos committed
1102
1103
1104
1105
1106
1107
1108


  Parameters:

    prefix (str): A string representing the root of the path in which the user
      objects are stored