experiments.py 31.5 KB
Newer Older
André Anjos's avatar
André Anjos committed
1
2
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
André Anjos's avatar
André Anjos committed
3

Samuel GAIST's avatar
Samuel GAIST committed
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
###################################################################################
#                                                                                 #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/               #
# Contact: beat.support@idiap.ch                                                  #
#                                                                                 #
# Redistribution and use in source and binary forms, with or without              #
# modification, are permitted provided that the following conditions are met:     #
#                                                                                 #
# 1. Redistributions of source code must retain the above copyright notice, this  #
# list of conditions and the following disclaimer.                                #
#                                                                                 #
# 2. Redistributions in binary form must reproduce the above copyright notice,    #
# this list of conditions and the following disclaimer in the documentation       #
# and/or other materials provided with the distribution.                          #
#                                                                                 #
# 3. Neither the name of the copyright holder nor the names of its contributors   #
# may be used to endorse or promote products derived from this software without   #
# specific prior written permission.                                              #
#                                                                                 #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED   #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE          #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE    #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL      #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR      #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER      #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,   #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE   #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.            #
#                                                                                 #
###################################################################################
André Anjos's avatar
André Anjos committed
35
36
37
38
39


import os
import logging
import glob
40
import click
André Anjos's avatar
André Anjos committed
41
42
43
44
import oset
import simplejson

from beat.core.experiment import Experiment
45
46
from beat.core.execution import DockerExecutor
from beat.core.execution import LocalExecutor
André Anjos's avatar
André Anjos committed
47
48
from beat.core.utils import NumpyJSONEncoder
from beat.core.data import CachedDataSource, load_data_index
49
from beat.core.dock import Host
50
51
52
from beat.core.hash import toPath
from beat.core.hash import hashDataset

53
from . import common
54
55
from .plotters import plot_impl as plotters_plot
from .plotters import pull_impl as plotters_pull
56
from .decorators import raise_on_error
57
58
from .click_helper import AliasedGroup

59
60

logger = logging.getLogger(__name__)
André Anjos's avatar
André Anjos committed
61
62


63
def run_experiment(configuration, name, force, use_docker, use_local, quiet):
64
    """Run experiments locally"""
65
66

    def load_result(executor):
67
        """Loads the result of an experiment, in a single go"""
68
69

        f = CachedDataSource()
70
71
72
73
74
75
76
77
        success = f.setup(
            os.path.join(executor.cache, executor.data["result"]["path"] + ".data"),
            executor.prefix,
        )

        if not success:
            raise RuntimeError("Failed to setup cached data source")

78
79
80
        data, start, end = f[0]
        return data

81
82
    def print_results(executor):
        data = load_result(executor)
83
84
85
        r = reindent(
            simplejson.dumps(data.as_dict(), indent=2, cls=NumpyJSONEncoder), 2
        )
86
87
        logger.info("  Results:\n%s", r)

88
    def reindent(s, n):
89
90
91
        """Re-indents output so it is more visible"""
        margin = n * " "
        return margin + ("\n" + margin).join(s.split("\n"))
92
93

    def simplify_time(s):
94
        """Re-writes the time so it is easier to understand it"""
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118

        minute = 60.0
        hour = 60 * minute
        day = 24 * hour

        if s <= minute:
            return "%.2f s" % s
        elif s <= hour:
            minutes = s // minute
            seconds = s - (minute * minutes)
            return "%d m %.2f s" % (minutes, seconds)
        elif s <= day:
            hours = s // hour
            minutes = (s - (hour * hours)) // minute
            seconds = s - (hour * hours + minute * minutes)
            return "%d h %d m %.2f s" % (hours, minutes, seconds)
        else:
            days = s // day
            hours = (s - (day * days)) // hour
            minutes = (s - (day * days + hour * hours)) // minute
            seconds = s - (day * days + hour * hours + minute * minutes)
            return "%d days %d h %d m %.2f s" % (days, hours, minutes, seconds)

    def simplify_size(s):
119
        """Re-writes the size so it is easier to understand it"""
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135

        kb = 1024.0
        mb = kb * kb
        gb = kb * mb
        tb = kb * gb

        if s <= kb:
            return "%d bytes" % s
        elif s <= mb:
            return "%.2f kilobytes" % (s / kb)
        elif s <= gb:
            return "%.2f megabytes" % (s / mb)
        elif s <= tb:
            return "%.2f gigabytes" % (s / gb)
        return "%.2f terabytes" % (s / tb)

136
    def index_experiment_databases(cache_path, experiment):
137
        for block_name, infos in experiment.datasets.items():
138
139
140
141
142
            view = infos["database"].view(infos["protocol"], infos["set"])
            filename = toPath(
                hashDataset(infos["database"].name, infos["protocol"], infos["set"]),
                suffix=".db",
            )
143
144
            database_index_path = os.path.join(cache_path, filename)
            if not os.path.exists(database_index_path):
145
146
147
148
                logger.info(
                    "Index for database %s not found, building it",
                    infos["database"].name,
                )
149
                view.index(database_index_path)
150
151
152
153
154
155

    dataformat_cache = {}
    database_cache = {}
    algorithm_cache = {}
    library_cache = {}

156
157
158
159
160
161
162
163
    experiment = Experiment(
        configuration.path,
        name,
        dataformat_cache,
        database_cache,
        algorithm_cache,
        library_cache,
    )
164
165
166
167

    if not experiment.valid:
        logger.error("Failed to load the experiment `%s':", name)
        for e in experiment.errors:
168
            logger.error("  * %s", e)
169
        return 1
André Anjos's avatar
André Anjos committed
170

171
172
173
    if not os.path.exists(configuration.cache):
        os.makedirs(configuration.cache)
        logger.info("Created cache path `%s'", configuration.cache)
174

175
    index_experiment_databases(configuration.cache, experiment)
André Anjos's avatar
André Anjos committed
176

177
    scheduled = experiment.setup()
André Anjos's avatar
André Anjos committed
178

179
    if use_docker:
180
181
182
183
184
185
186
187
188
189
        # load existing environments
        host = Host(raise_on_errors=False)

    # can we execute it?
    for key, value in scheduled.items():

        # checks and sets-up executable
        executable = None  # use the default

        if use_docker:
190
191
            env = value["configuration"]["environment"]
            search_key = "%s (%s)" % (env["name"], env["version"])
192
            if search_key not in host:
193
194
195
196
197
198
                logger.error(
                    "Cannot execute block `%s' on environment `%s': "
                    "environment was not found' - please install it",
                    key,
                    search_key,
                )
199
200
201
                return 1

        if use_docker:
202
203
204
205
206
207
208
209
210
211
            executor = DockerExecutor(
                host,
                configuration.path,
                value["configuration"],
                configuration.cache,
                dataformat_cache,
                database_cache,
                algorithm_cache,
                library_cache,
            )
212
        else:
213
214
215
216
217
218
219
220
221
222
            executor = LocalExecutor(
                configuration.path,
                value["configuration"],
                configuration.cache,
                dataformat_cache,
                database_cache,
                algorithm_cache,
                library_cache,
                configuration.database_paths,
            )
223
224

        if not executor.valid:
225
            logger.error("Failed to load the execution information for `%s':", key)
226
            for e in executor.errors:
227
                logger.error("  * %s", e)
228
229
230
            return 1

        if executor.outputs_exist and not force:
231
232
233
234
235
            logger.info(
                "Skipping execution of `%s' for block `%s' " "- outputs exist",
                executor.algorithm.name,
                key,
            )
236
            if executor.analysis and not quiet:
237
238
                logger.extra("  Outputs produced:")
                print_results(executor)
239
240
            continue

241
        logger.info("Running `%s' for block `%s'", executor.algorithm.name, key)
242
243
244
245
246
247
248
249
        if executable is not None:
            logger.extra("  -> using executable at `%s'", executable)
        else:
            logger.extra("  -> using fallback (default) environment")

        with executor:
            result = executor.process()

250
        if result["status"] != 0:
251
            logger.error("Block did not execute properly - outputs were reset")
252
253
254
255
256
257
258
259
260
            logger.error("  Standard output:\n%s", reindent(result["stdout"], 4))
            logger.error("  Standard error:\n%s", reindent(result["stderr"], 4))
            logger.error(
                "  Captured user error:\n%s", reindent(result["user_error"], 4)
            )
            logger.error(
                "  Captured system error:\n%s", reindent(result["system_error"], 4)
            )
            logger.extra("  Environment: %s" % "default environment")
261
262
            return 1
        elif use_docker:
263
264
265
            stats = result["statistics"]
            cpu_stats = stats["cpu"]
            data_stats = stats["data"]
266

267
            cpu_total = cpu_stats["total"]
268
269
270
271
            # Likely means that GPU was used
            if not cpu_total:
                cpu_total = 1.0

272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
            logger.extra(
                "  CPU time (user, system, total, percent): " "%s, %s, %s, %d%%",
                simplify_time(cpu_stats["user"]),
                simplify_time(cpu_stats["system"]),
                simplify_time(cpu_total),
                100.0 * (cpu_stats["user"] + cpu_stats["system"]) / cpu_total,
            )
            logger.extra("  Memory usage: %s", simplify_size(stats["memory"]["rss"]))
            logger.extra(
                "  Cached input read: %s, %s",
                simplify_time(data_stats["time"]["read"]),
                simplify_size(data_stats["volume"]["read"]),
            )
            logger.extra(
                "  Cached output write: %s, %s",
                simplify_time(data_stats["time"]["write"]),
                simplify_size(data_stats["volume"]["write"]),
            )
            logger.extra(
                "  Communication time: %s (%d%%)",
                simplify_time(data_stats["network"]["wait_time"]),
                100.0 * data_stats["network"]["wait_time"] / cpu_total,
            )
295
        else:
296
            logger.extra("  Environment: %s" % "local environment")
297

298
299
300
        if not quiet:
            if executor.analysis:
                print_results(executor)
301

302
303
            logger.extra("  Outputs produced:")
            if executor.analysis:
304
                logger.extra("    * %s", executor.data["result"]["path"])
305
            else:
306
307
                for name, details in executor.data["outputs"].items():
                    logger.extra("    * %s", details["path"])
308
        else:
309
            logger.info("Done")
310
311

    return 0
André Anjos's avatar
André Anjos committed
312
313


314
def caches_impl(configuration, name, ls, delete, checksum):
315
    """List all cache files involved in this experiment"""
André Anjos's avatar
André Anjos committed
316

317
318
319
320
    dataformat_cache = {}
    database_cache = {}
    algorithm_cache = {}
    library_cache = {}
André Anjos's avatar
André Anjos committed
321

322
323
324
325
326
327
328
329
    experiment = Experiment(
        configuration.path,
        name,
        dataformat_cache,
        database_cache,
        algorithm_cache,
        library_cache,
    )
André Anjos's avatar
André Anjos committed
330

331
332
333
    if not experiment.valid:
        logger.error("Failed to load the experiment `%s':", name)
        for e in experiment.errors:
334
            logger.error("  * %s", e)
335
        return 1
André Anjos's avatar
André Anjos committed
336

337
    scheduled = experiment.setup()
André Anjos's avatar
André Anjos committed
338

339
    block_list = []
340
    for key, value in scheduled.items():
341
        block = {
342
343
344
345
            "name": key,
            "algorithm": value["configuration"]["algorithm"],
            "is_analyser": False,
            "paths": [],
346
347
        }

348
349
350
        if "outputs" in value["configuration"]:  # normal block
            for name, data in value["configuration"]["outputs"].items():
                block["paths"].append(data["path"])
351
        else:  # analyzer
352
353
            block["is_analyser"] = True
            block["paths"].append(value["configuration"]["result"]["path"])
André Anjos's avatar
André Anjos committed
354

355
        block_list.append(block)
André Anjos's avatar
André Anjos committed
356

357
    for block in block_list:
358
359
360
        block_type = "analyzer" if block["is_analyser"] else "algorithm"
        logger.info("block: `%s'", block["name"])
        logger.info("  %s: `%s'", block_type, block["algorithm"])
André Anjos's avatar
André Anjos committed
361

362
        for path in block["paths"]:
363
364
365
            # prefix cache path
            path = os.path.join(configuration.cache, path)
            logger.info("  output: `%s'", path)
André Anjos's avatar
André Anjos committed
366

367
            if ls:
368
369
                for file in glob.glob(path + ".*"):
                    logger.info("    %s" % file)
André Anjos's avatar
André Anjos committed
370

371
            if delete:
372
                for file in glob.glob(path + ".*"):
373
374
                    logger.info("removing `%s'...", file)
                    os.unlink(file)
André Anjos's avatar
André Anjos committed
375

376
                common.recursive_rmdir_if_empty(
377
378
                    os.path.dirname(path), configuration.cache
                )
André Anjos's avatar
André Anjos committed
379

380
            if checksum:
381
382
                if not load_data_index(configuration.cache, path + ".data"):
                    logger.error("Failed to load data index for {}".format(path))
383
                logger.info("index for `%s' can be loaded and checksums", path)
384

385
    return 0
André Anjos's avatar
André Anjos committed
386
387


388
def pull_impl(webapi, prefix, names, force, indentation, format_cache):
389
    """Copies experiments (and required toolchains/algorithms) from the server.
André Anjos's avatar
André Anjos committed
390

391
    Parameters:
André Anjos's avatar
André Anjos committed
392

393
394
      webapi (object): An instance of our WebAPI class, prepared to access the
        BEAT server of interest
André Anjos's avatar
André Anjos committed
395

396
397
      prefix (str): A string representing the root of the path in which the
        user objects are stored
André Anjos's avatar
André Anjos committed
398

André Anjos's avatar
André Anjos committed
399
400
401
402
403
      names (:py:class:`list`): A list of strings, each representing the unique
        relative path of the objects to retrieve or a list of usernames from
        which to retrieve objects. If the list is empty, then we pull all
        available objects of a given type. If no user is set, then pull all
        public objects of a given type.
André Anjos's avatar
André Anjos committed
404

405
406
      force (bool): If set to ``True``, then overwrites local changes with the
        remotely retrieved copies.
André Anjos's avatar
André Anjos committed
407

408
409
410
      indentation (int): The indentation level, useful if this function is
        called recursively while downloading different object types. This is
        normally set to ``0`` (zero).
André Anjos's avatar
André Anjos committed
411
412


413
    Returns:
André Anjos's avatar
André Anjos committed
414

415
416
417
      int: Indicating the exit status of the command, to be reported back to
        the calling process. This value should be zero if everything works OK,
        otherwise, different than zero (POSIX compliance).
André Anjos's avatar
André Anjos committed
418

419
    """
André Anjos's avatar
André Anjos committed
420

421
422
    from .algorithms import pull_impl as algorithms_pull
    from .databases import pull_impl as databases_pull
André Anjos's avatar
André Anjos committed
423

424
425
426
427
428
429
430
431
432
433
434
435
    if indentation == 0:
        indentation = 4

    status, names = common.pull(
        webapi,
        prefix,
        "experiment",
        names,
        ["declaration", "description"],
        force,
        indentation,
    )
André Anjos's avatar
André Anjos committed
436

437
    if status != 0:
438
        logger.error("could not find any matching experiments - widen your search")
439
        return status
André Anjos's avatar
André Anjos committed
440

441
442
443
444
445
446
447
448
449
450
451
    # see what dataformats one needs to pull
    databases = oset.oset()
    toolchains = oset.oset()
    algorithms = oset.oset()
    for name in names:
        try:
            obj = Experiment(prefix, name)
            if obj.toolchain:
                toolchains.add(obj.toolchain.name)
            databases |= obj.databases.keys()
            algorithms |= obj.algorithms.keys()
André Anjos's avatar
André Anjos committed
452

453
454
        except Exception as e:
            logger.error("loading `%s': %s...", name, str(e))
André Anjos's avatar
André Anjos committed
455

456
457
458
    # downloads any formats to which we depend on
    format_cache = {}
    library_cache = {}
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
    tc_status, _ = common.pull(
        webapi,
        prefix,
        "toolchain",
        toolchains,
        ["declaration", "description"],
        force,
        indentation,
    )
    db_status = databases_pull(
        webapi, prefix, databases, force, indentation, format_cache
    )
    algo_status = algorithms_pull(
        webapi, prefix, algorithms, force, indentation, format_cache, library_cache
    )
André Anjos's avatar
André Anjos committed
474

475
    return status + tc_status + db_status + algo_status
André Anjos's avatar
André Anjos committed
476
477


478
479
480
481
482
483
484
485
486
487
488
489
def plot_impl(
    webapi,
    configuration,
    prefix,
    names,
    remote_results,
    show,
    force,
    indentation,
    format_cache,
    outputfolder=None,
):
490
491
492
493
494
495
496
    """Plots experiments from the server.

    Parameters:

      webapi (object): An instance of our WebAPI class, prepared to access the
        BEAT server of interest

497
498
499
      configuration (object): An instance of the configuration, to access the
        BEAT server and current configuration for information

500
501
502
      prefix (str): A string representing the root of the path in which the
        user objects are stored

503
      names (:py:class:`list`): A list of strings, each representing the unique relative
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
        path of the objects to retrieve or a list of usernames from which to
        retrieve objects. If the list is empty, then we pull all available
        objects of a given type. If no user is set, then pull all public
        objects of a given type.

      remote_results(bool): If set to ``True``, then fetch results data
        for the experiments from the server.

      force (bool): If set to ``True``, then overwrites local changes with the
        remotely retrieved copies.

      indentation (int): The indentation level, useful if this function is
        called recursively while downloading different object types. This is
        normally set to ``0`` (zero).

519
520
      outputfolder (str): A string representing the path in which the
        experiments plot will be stored
521
522
523
524
525
526
527
528
529
530

    Returns:

      int: Indicating the exit status of the command, to be reported back to
        the calling process. This value should be zero if everything works OK,
        otherwise, different than zero (POSIX compliance).

    """

    status = 0
531
532
533
534
    RESULTS_SIMPLE_TYPE_NAMES = ("int32", "float32", "bool", "string")

    if indentation == 0:
        indentation = 4
535

536
537
538
539
540
541
542
543
544
    if remote_results:
        if outputfolder is None:
            output_folder = configuration.path
        else:
            # check if directory exists else create
            if not os.path.isdir(outputfolder):
                os.mkdir(os.path.join(configuration.path, outputfolder))
            output_folder = os.path.join(configuration.path, outputfolder)

545
    for name in names:
546
547
        if not remote_results:
            if outputfolder is None:
548
549
550
551
552
                output_folder = os.path.join(
                    configuration.path,
                    common.TYPE_PLURAL["experiment"],
                    name.rsplit("/", 1)[0],
                )
553
554
555
556
557
558
            else:
                # check if directory exists else create
                if not os.path.isdir(outputfolder):
                    os.mkdir(os.path.join(configuration.path, outputfolder))
                output_folder = os.path.join(configuration.path, outputfolder)

559
        check_plottable = False
560
        if not os.path.exists(configuration.cache) or remote_results:
561
562
563
564
565
566
            experiment = simplejson.loads(
                simplejson.dumps(
                    common.fetch_object(webapi, "experiment", name, ["results"])
                )
            )
            results = experiment["results"]["analysis"]
567
568
            for key, value in results.iteritems():
                # remove non plottable results
569
570
                if value["type"] not in RESULTS_SIMPLE_TYPE_NAMES:
                    output_name = name.rsplit("/", 1)[1] + "_" + key + ".png"
571
                    output_name = os.path.join(output_folder, output_name)
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
                    pl_status = plotters_pull(
                        webapi,
                        configuration.path,
                        [value["type"]],
                        force,
                        indentation,
                        {},
                    )
                    plot_status = plotters_plot(
                        webapi,
                        configuration.path,
                        [value["type"]],
                        show,
                        False,
                        False,
                        value["value"],
                        output_name,
                        None,
                        indentation,
                        format_cache,
                    )
593
594
                    status += pl_status
                    status += plot_status
595
                    check_plottable = True
596
        else:
597
            # make sure experiment exists locally or pull it
598
599
600
            pull_impl(
                webapi, configuration.path, [name], force, indentation, format_cache
            )
601
602
603
604
605
606
607

            # get information from cache
            dataformat_cache = {}
            database_cache = {}
            algorithm_cache = {}
            library_cache = {}

608
609
610
611
612
613
614
615
            experiment = Experiment(
                configuration.path,
                name,
                dataformat_cache,
                database_cache,
                algorithm_cache,
                library_cache,
            )
616
617
618

            scheduled = experiment.setup()
            for key, value in scheduled.items():
619
620
621
622
623
624
625
626
627
628
629
630
                executor = LocalExecutor(
                    configuration.path,
                    value["configuration"],
                    configuration.cache,
                    dataformat_cache,
                    database_cache,
                    algorithm_cache,
                    library_cache,
                    configuration.database_paths,
                )

                if "result" in executor.data:
631
                    f = CachedDataSource()
632
633
634
635
636
637
638
639
640
                    success = f.setup(
                        os.path.join(
                            executor.cache, executor.data["result"]["path"] + ".data"
                        ),
                        executor.prefix,
                    )
                    if not success:
                        raise RuntimeError("Failed to setup cached data source")

641
642
643
644
                    data, start, end = f[0]

                    for the_data in data.as_dict():
                        attr = getattr(data, the_data)
645
646
                        if attr.__class__.__name__.startswith("plot"):
                            datatype = attr.__class__.__name__.replace("_", "/")
647
648
                            # remove non plottable results
                            if datatype not in RESULTS_SIMPLE_TYPE_NAMES:
649
650
651
                                output_name = (
                                    name.rsplit("/", 1)[1] + "_" + the_data + ".png"
                                )
652
                                output_name = os.path.join(output_folder, output_name)
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
                                pl_status = plotters_pull(
                                    webapi,
                                    configuration.path,
                                    [datatype],
                                    force,
                                    indentation,
                                    {},
                                )
                                plot_status = plotters_plot(
                                    webapi,
                                    configuration.path,
                                    [datatype],
                                    show,
                                    False,
                                    False,
                                    data.as_dict()[the_data],
                                    output_name,
                                    None,
                                    indentation,
                                    format_cache,
                                )
674
675
                                status += pl_status
                                status += plot_status
676
677
                                check_plottable = True
        if not check_plottable:
678
            print("Experiments results are not plottable")
679
680
681
682

    return status


683
@click.group(cls=AliasedGroup)
684
685
686
687
688
689
690
@click.pass_context
def experiments(ctx):
    """experiments commands"""
    pass


@experiments.command()
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
@click.argument("name", nargs=1)
@click.option(
    "--force", help="Performs operation regardless of conflicts", is_flag=True
)
@click.option(
    "--docker",
    help="Uses the docker executor to execute the "
    "experiment using docker containers",
    is_flag=True,
)
@click.option(
    "--local",
    help="Uses the local executor to execute the "
    "experiment on the local machine (default)",
    default=True,
    is_flag=True,
)
@click.option("--quiet", help="Be less verbose", is_flag=True)
709
@click.pass_context
710
@raise_on_error
711
def run(ctx, name, force, docker, local, quiet):
712
713
    """ Runs an experiment locally"""
    config = ctx.meta.get("config")
714
    return run_experiment(config, name, force, docker, local, quiet)
715
716
717


@experiments.command()
718
719
720
721
722
723
724
725
726
727
728
@click.argument("name", nargs=1)
@click.option(
    "--list", help="List cache files matching output if they exist", is_flag=True
)
@click.option(
    "--delete",
    help="Delete cache files matching output if they "
    "exist (also, recursively deletes empty directories)",
    is_flag=True,
)
@click.option("--checksum", help="Checksums indexes for cache files", is_flag=True)
729
@click.pass_context
730
@raise_on_error
731
def caches(ctx, name, list, delete, checksum):
732
733
    """Lists all cache files used by this experiment"""
    config = ctx.meta.get("config")
734
    return caches_impl(config, name, list, delete, checksum)
735
736
737


@experiments.command()
738
739
740
@click.option(
    "--remote", help="Only acts on the remote copy of the list.", is_flag=True
)
741
@click.pass_context
742
@raise_on_error
743
def list(ctx, remote):
744
    """Lists all the experiments available on the platform.
745
746
747
748

    To list all existing experiments on your local prefix:

        $ beat experiments list
749
750
    """
    config = ctx.meta.get("config")
751
    if remote:
752
        with common.make_webapi(config) as webapi:
753
            return common.display_remote_list(webapi, "experiment")
754
    else:
755
        return common.display_local_list(config.path, "experiment")
756
757
758


@experiments.command()
759
@click.argument("names", nargs=-1)
760
@click.pass_context
761
@raise_on_error
762
def path(ctx, names):
763
    """Displays local path of experiment files
764
765
766

  Example:
    $ beat experiments path xxx
767
768
  """
    return common.display_local_path(ctx.meta["config"].path, "experiment", names)
769
770
771


@experiments.command()
772
@click.argument("name", nargs=1)
773
@click.pass_context
774
@raise_on_error
775
def edit(ctx, name):
776
    """Edit local experiment file
777
778
779

  Example:
    $ beat experiments edit xxx
780
781
782
783
  """
    return common.edit_local_file(
        ctx.meta["config"].path, ctx.meta["config"].editor, "experiment", name
    )
784
785
786


@experiments.command()
787
@click.argument("names", nargs=-1)
788
@click.pass_context
789
@raise_on_error
790
def check(ctx, names):
791
    """Checks a local experiment for validity.
792
793

    $ beat experiments check xxx
794
795
796
    """
    config = ctx.meta.get("config")
    return common.check(config.path, "experiment", names)
797
798
799


@experiments.command()
800
801
802
803
@click.argument("names", nargs=-1)
@click.option(
    "--force", help="Performs operation regardless of conflicts", is_flag=True
)
804
@click.pass_context
805
@raise_on_error
806
def pull(ctx, names, force):
807
    """Downloads the specified experiments from the server.
808
809

       $ beat experiments pull xxx.
810
811
    """
    config = ctx.meta.get("config")
812
    with common.make_webapi(config) as webapi:
813
        return pull_impl(webapi, config.path, names, force, 0, {})
814
815
816


@experiments.command()
817
818
819
820
821
822
823
824
825
@click.argument("names", nargs=-1)
@click.option(
    "--force", help="Performs operation regardless of conflicts", is_flag=True
)
@click.option(
    "--dry-run",
    help="Doesn't really perform the task, just " "comments what would do",
    is_flag=True,
)
826
@click.pass_context
827
@raise_on_error
828
def push(ctx, names, force, dry_run):
829
    """Uploads experiments to the server.
830
831
832

    Example:
      $ beat experiments push --dry-run yyy
833
834
    """
    config = ctx.meta.get("config")
835
836
    with common.make_webapi(config) as webapi:
        return common.push(
837
838
839
840
841
842
843
844
845
            webapi,
            config.path,
            "experiment",
            names,
            ["name", "declaration", "toolchain", "description"],
            {},
            force,
            dry_run,
            0,
846
847
848
849
        )


@experiments.command()
850
@click.argument("name", nargs=1)
851
@click.pass_context
852
@raise_on_error
853
def diff(ctx, name):
854
    """Shows changes between the local dataformat and the remote version.
855
856
857

    Example:
      $ beat experiments diff xxx
858
859
    """
    config = ctx.meta.get("config")
860
861
    with common.make_webapi(config) as webapi:
        return common.diff(
862
            webapi, config.path, "experiment", name, ["declaration", "description"]
863
864
865
866
867
        )


@experiments.command()
@click.pass_context
868
@raise_on_error
869
def status(ctx):
870
    """Shows (editing) status for all available experiments.
871
872
873

    Example:
      $ beat experiments status
874
875
    """
    config = ctx.meta.get("config")
876
    with common.make_webapi(config) as webapi:
877
        return common.status(webapi, config.path, "experiment")[0]
878
879
880


@experiments.command()
881
882
@click.argument("src", nargs=1)
@click.argument("dst", nargs=1)
883
@click.pass_context
884
@raise_on_error
885
def fork(ctx, src, dst):
886
    """Forks a local experiment.
887
888

    $ beat experiments fork xxx yyy
889
890
891
    """
    config = ctx.meta.get("config")
    return common.fork(config.path, "experiment", src, dst)
892
893
894


@experiments.command()
895
896
897
898
@click.argument("names", nargs=-1)
@click.option(
    "--remote", help="Only acts on the remote copy of the experiment", is_flag=True
)
899
@click.pass_context
900
@raise_on_error
901
def rm(ctx, names, remote):
902
    """Deletes a local experiment (unless --remote is specified).
903
904

    $ beat experiments rm xxx
905
906
    """
    config = ctx.meta.get("config")
907
    if remote:
908
        with common.make_webapi(config) as webapi:
909
            return common.delete_remote(webapi, "experiment", names)
910
    else:
911
        return common.delete_local(config.path, "experiment", names)
912
913
914


@experiments.command()
915
916
917
918
919
920
@click.argument("names", nargs=-1)
@click.option(
    "--path",
    help="Use path to write files to disk (instead of the " "current directory)",
    type=click.Path(),
)
921
@click.pass_context
922
@raise_on_error
923
def draw(ctx, names, path):
924
925
926
    """Creates a visual representation of the experiment."""
    config = ctx.meta.get("config")
    return common.dot_diagram(config.path, "experiment", names, path, [])
927
928
929


@experiments.command()
930
931
932
933
934
935
936
937
938
@click.argument("names", nargs=-1)
@click.option(
    "--force", help="Performs operation regardless of conflicts", is_flag=True
)
@click.option(
    "--remote", help="Only acts on the remote copy of the experiment", is_flag=True
)
@click.option("--show", help="Show...", is_flag=True)
@click.option("--output-folder", help="<folder>", type=click.Path(exists=True))
939
@click.pass_context
940
@raise_on_error
941
def plot(ctx, names, force, remote, show, output_folder):
942
943
    """Plots output images of the experiment."""
    config = ctx.meta.get("config")
944
945
    with common.make_webapi(config) as webapi:
        return plot_impl(
946
947
948
949
950
951
952
953
954
955
            webapi,
            config,
            "experiment",
            names,
            remote,
            show,
            force,
            0,
            {},
            output_folder,
956
        )