databases.py 24 KB
Newer Older
André Anjos's avatar
André Anjos committed
1 2 3
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
###################################################################################
#                                                                                 #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/               #
# Contact: beat.support@idiap.ch                                                  #
#                                                                                 #
# Redistribution and use in source and binary forms, with or without              #
# modification, are permitted provided that the following conditions are met:     #
#                                                                                 #
# 1. Redistributions of source code must retain the above copyright notice, this  #
# list of conditions and the following disclaimer.                                #
#                                                                                 #
# 2. Redistributions in binary form must reproduce the above copyright notice,    #
# this list of conditions and the following disclaimer in the documentation       #
# and/or other materials provided with the distribution.                          #
#                                                                                 #
# 3. Neither the name of the copyright holder nor the names of its contributors   #
# may be used to endorse or promote products derived from this software without   #
# specific prior written permission.                                              #
#                                                                                 #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED   #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE          #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE    #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL      #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR      #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER      #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,   #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE   #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.            #
#                                                                                 #
###################################################################################

André Anjos's avatar
André Anjos committed
36 37 38

import glob
import logging
Samuel GAIST's avatar
Samuel GAIST committed
39 40 41 42
import os
import random

import click
André Anjos's avatar
André Anjos committed
43
import simplejson
Samuel GAIST's avatar
Samuel GAIST committed
44
import zmq
André Anjos's avatar
André Anjos committed
45

46 47 48
from beat.core import dock
from beat.core import inputs
from beat.core import utils
Samuel GAIST's avatar
Samuel GAIST committed
49 50 51 52 53
from beat.core.data import RemoteDataSource
from beat.core.database import Database
from beat.core.hash import hashDataset
from beat.core.hash import toPath
from beat.core.utils import NumpyJSONEncoder
André Anjos's avatar
André Anjos committed
54

55
from . import commands
Samuel GAIST's avatar
Samuel GAIST committed
56
from . import common
57
from .click_helper import AliasedGroup
58 59
from .click_helper import AssetCommand
from .click_helper import AssetInfo
Samuel GAIST's avatar
Samuel GAIST committed
60
from .decorators import raise_on_error
André Anjos's avatar
André Anjos committed
61

Samuel GAIST's avatar
Samuel GAIST committed
62
logger = logging.getLogger(__name__)
André Anjos's avatar
André Anjos committed
63

64

65 66
CMD_DB_INDEX = "index"
CMD_VIEW_OUTPUTS = "databases_provider"
67 68


Samuel GAIST's avatar
Samuel GAIST committed
69
# ----------------------------------------------------------
70 71 72 73


def load_database_sets(configuration, database_name):
    # Process the name of the database
74
    parts = database_name.split("/")
75 76

    if len(parts) == 2:
77 78 79
        db_name = os.path.join(*parts[:2])
        protocol_filter = None
        set_filter = None
80 81

    elif len(parts) == 3:
82 83 84
        db_name = os.path.join(*parts[:2])
        protocol_filter = parts[2]
        set_filter = None
85 86

    elif len(parts) == 4:
87 88 89
        db_name = os.path.join(*parts[:2])
        protocol_filter = parts[2]
        set_filter = parts[3]
90 91

    else:
92 93 94 95 96 97
        logger.error(
            "Database specification should have the format "
            "`<database>/<version>/[<protocol>/[<set>]]', the value "
            "you passed (%s) is not valid",
            database_name,
        )
98
        return (None, None)
99 100 101

    # Load the dataformat
    dataformat_cache = {}
102
    database = Database(configuration.path, db_name, dataformat_cache)
103
    if not database.valid:
104 105
        logger.error("Failed to load the database `%s':", db_name)
        for e in database.errors:
106
            logger.error("  * %s", e)
107
        return (None, None, None)
108 109 110 111 112

    # Filter the protocols
    protocols = database.protocol_names

    if protocol_filter is not None:
113
        if protocol_filter not in protocols:
114 115 116 117 118 119 120
            logger.error(
                "The database `%s' does not have the protocol `%s' - "
                "choose one of `%s'",
                db_name,
                protocol_filter,
                ", ".join(protocols),
            )
121

122
            return (None, None, None)
123

124
        protocols = [protocol_filter]
125 126 127 128 129

    # Filter the sets
    loaded_sets = []

    for protocol_name in protocols:
130
        sets = database.set_names(protocol_name)
131

132 133
        if set_filter is not None:
            if set_filter not in sets:
134 135 136 137 138 139 140 141
                logger.error(
                    "The database/protocol `%s/%s' does not have the "
                    "set `%s' - choose one of `%s'",
                    db_name,
                    protocol_name,
                    set_filter,
                    ", ".join(sets),
                )
142
                return (None, None, None)
143

144
            sets = [z for z in sets if z == set_filter]
145

146 147 148 149 150 151
        loaded_sets.extend(
            [
                (protocol_name, set_name, database.set(protocol_name, set_name))
                for set_name in sets
            ]
        )
152 153 154 155

    return (db_name, database, loaded_sets)


Samuel GAIST's avatar
Samuel GAIST committed
156
# ----------------------------------------------------------
157 158


159 160 161 162 163 164 165 166 167 168 169 170 171
def start_db_container(
    configuration,
    cmd,
    host,
    db_name,
    protocol_name,
    set_name,
    database,
    db_set,
    excluded_outputs=None,
    uid=None,
    db_root=None,
):
172 173 174 175 176 177

    input_list = inputs.InputList()

    input_group = inputs.InputGroup(set_name, restricted_access=False)
    input_list.add(input_group)

178
    db_configuration = {"inputs": {}, "channel": set_name}
179 180

    if uid is not None:
181
        db_configuration["datasets_uid"] = uid
182 183

    if db_root is not None:
184
        db_configuration["datasets_root_path"] = db_root
185

186
    for output_name, dataformat_name in db_set["outputs"].items():
Samuel GAIST's avatar
Samuel GAIST committed
187
        if excluded_outputs is not None and output_name in excluded_outputs:
188 189
            continue

190
        dataset_hash = hashDataset(db_name, protocol_name, set_name)
191
        db_configuration["inputs"][output_name] = dict(
192 193 194 195 196 197
            database=db_name,
            protocol=protocol_name,
            set=set_name,
            output=output_name,
            channel=set_name,
            hash=dataset_hash,
198
            path=toPath(dataset_hash, ".db"),
199 200 201 202
        )

    db_tempdir = utils.temporary_directory()

203
    with open(os.path.join(db_tempdir, "configuration.json"), "wt") as f:
204 205
        simplejson.dump(db_configuration, f, indent=4)

206
    tmp_prefix = os.path.join(db_tempdir, "prefix")
207 208 209
    if not os.path.exists(tmp_prefix):
        os.makedirs(tmp_prefix)

210
    database.export(tmp_prefix)
211 212

    if db_root is None:
213
        json_path = os.path.join(tmp_prefix, "databases", db_name + ".json")
214

215
        with open(json_path, "r") as f:
216 217
            db_data = simplejson.load(f)

218 219
        database_path = db_data["root_folder"]
        db_data["root_folder"] = os.path.join("/databases", db_name)
220

221
        with open(json_path, "w") as f:
222 223 224 225
            simplejson.dump(db_data, f, indent=4)

    try:
        db_envkey = host.db2docker([db_name])
226 227 228 229 230 231
    except Exception:
        raise RuntimeError(
            "No environment found for the database `%s' "
            "- available environments are %s"
            % (db_name, ", ".join(host.db_environments.keys()))
        )
232 233 234

    # Creation of the container
    # Note: we only support one databases image loaded at the same time
235 236
    CONTAINER_PREFIX = "/beat/prefix"
    CONTAINER_CACHE = "/beat/cache"
237

238
    database_port = random.randint(51000, 60000)  # nosec just getting a free port
239
    if cmd == CMD_VIEW_OUTPUTS:
240 241
        db_cmd = [
            cmd,
242
            "0.0.0.0:{}".format(database_port),
243
            CONTAINER_PREFIX,
244
            CONTAINER_CACHE,
245
        ]
246
    else:
247 248 249 250 251 252
        db_cmd = [
            cmd,
            CONTAINER_PREFIX,
            CONTAINER_CACHE,
            db_name,
            protocol_name,
253
            set_name,
254
        ]
255 256

    databases_container = host.create_container(db_envkey, db_cmd)
257 258
    databases_container.uid = uid

259
    if cmd == CMD_VIEW_OUTPUTS:
260 261 262
        databases_container.add_port(database_port, database_port, host_address=host.ip)
        databases_container.add_volume(db_tempdir, "/beat/prefix")
        databases_container.add_volume(configuration.cache, "/beat/cache")
263
    else:
264 265 266 267
        databases_container.add_volume(tmp_prefix, "/beat/prefix")
        databases_container.add_volume(
            configuration.cache, "/beat/cache", read_only=False
        )
268 269

    # Specify the volumes to mount inside the container
270
    if "datasets_root_path" not in db_configuration:
271
        databases_container.add_volume(
272 273
            database_path, os.path.join("/databases", db_name)
        )
274
    else:
275 276 277 278
        databases_container.add_volume(
            db_configuration["datasets_root_path"],
            db_configuration["datasets_root_path"],
        )
279 280 281 282

    # Start the container
    host.start(databases_container)

283
    if cmd == CMD_VIEW_OUTPUTS:
284 285 286
        # Communicate with container
        zmq_context = zmq.Context()
        db_socket = zmq_context.socket(zmq.PAIR)
287
        db_address = "tcp://{}:{}".format(host.ip, database_port)
288
        db_socket.connect(db_address)
289

290 291
        for output_name, dataformat_name in db_set["outputs"].items():
            if excluded_outputs is not None and output_name in excluded_outputs:
292
                continue
293

294
            data_source = RemoteDataSource()
295 296 297
            data_source.setup(
                db_socket, output_name, dataformat_name, configuration.path
            )
298

299 300 301
            input_ = inputs.Input(
                output_name, database.dataformats[dataformat_name], data_source
            )
Samuel GAIST's avatar
Samuel GAIST committed
302
            input_group.add(input_)
303

304
        return (databases_container, db_socket, zmq_context, input_list)
305 306

    return databases_container
307 308


Samuel GAIST's avatar
Samuel GAIST committed
309
# ----------------------------------------------------------
310 311


312
def pull_impl(webapi, prefix, names, force, indentation, format_cache):
313
    """Copies databases (and required dataformats) from the server.
André Anjos's avatar
André Anjos committed
314

315
    Parameters:
André Anjos's avatar
André Anjos committed
316

317 318
      webapi (object): An instance of our WebAPI class, prepared to access the
        BEAT server of interest
André Anjos's avatar
André Anjos committed
319

Samuel GAIST's avatar
Samuel GAIST committed
320 321
      prefix (str): A string representing the root of the path in which the
        user objects are stored
André Anjos's avatar
André Anjos committed
322

André Anjos's avatar
André Anjos committed
323 324 325 326 327
      names (:py:class:`list`): A list of strings, each representing the unique
        relative path of the objects to retrieve or a list of usernames from
        which to retrieve objects. If the list is empty, then we pull all
        available objects of a given type. If no user is set, then pull all
        public objects of a given type.
André Anjos's avatar
André Anjos committed
328

329 330
      force (bool): If set to ``True``, then overwrites local changes with the
        remotely retrieved copies.
André Anjos's avatar
André Anjos committed
331

Samuel GAIST's avatar
Samuel GAIST committed
332 333 334
      indentation (int): The indentation level, useful if this function is
        called recursively while downloading different object types. This is
        normally set to ``0`` (zero).
André Anjos's avatar
André Anjos committed
335

336 337
      format_cache (dict): A dictionary containing all dataformats already
        downloaded.
André Anjos's avatar
André Anjos committed
338 339


340
    Returns:
André Anjos's avatar
André Anjos committed
341

Samuel GAIST's avatar
Samuel GAIST committed
342 343
      int: Indicating the exit status of the command, to be reported back to
        the calling process. This value should be zero if everything works OK,
344
        otherwise, different than zero (POSIX compliance).
André Anjos's avatar
André Anjos committed
345

346
    """
André Anjos's avatar
André Anjos committed
347

348
    from .dataformats import pull_impl as dataformats_pull
André Anjos's avatar
André Anjos committed
349

350 351 352 353 354 355 356 357 358
    status, names = common.pull(
        webapi,
        prefix,
        "database",
        names,
        ["declaration", "code", "description"],
        force,
        indentation,
    )
André Anjos's avatar
André Anjos committed
359

360 361 362 363 364
    # see what dataformats one needs to pull
    dataformats = []
    for name in names:
        obj = Database(prefix, name)
        dataformats.extend(obj.dataformats.keys())
André Anjos's avatar
André Anjos committed
365

366
    # downloads any formats to which we depend on
367 368 369
    df_status = dataformats_pull(
        webapi, prefix, dataformats, force, indentation + 2, format_cache
    )
André Anjos's avatar
André Anjos committed
370

371
    return status + df_status
André Anjos's avatar
André Anjos committed
372 373


Samuel GAIST's avatar
Samuel GAIST committed
374
# ----------------------------------------------------------
André Anjos's avatar
André Anjos committed
375 376


377
def index_outputs(configuration, names, uid=None, db_root=None, docker=False):
André Anjos's avatar
André Anjos committed
378

379
    names = common.make_up_local_list(configuration.path, "database", names)
380
    retcode = 0
André Anjos's avatar
André Anjos committed
381

Philip ABBET's avatar
Philip ABBET committed
382
    if docker:
383
        host = dock.Host(raise_on_errors=False)
André Anjos's avatar
André Anjos committed
384

385
    for database_name in names:
386
        logger.info("Indexing database %s...", database_name)
André Anjos's avatar
André Anjos committed
387

388
        (db_name, database, sets) = load_database_sets(configuration, database_name)
389 390 391
        if database is None:
            retcode += 1
            continue
André Anjos's avatar
André Anjos committed
392

393 394
        for protocol_name, set_name, db_set in sets:
            if not docker:
395 396 397
                try:
                    view = database.view(protocol_name, set_name)
                except SyntaxError as error:
398 399
                    logger.error("Failed to load the database `%s':", database_name)
                    logger.error("  * Syntax error: %s", error)
400
                    view = None
André Anjos's avatar
André Anjos committed
401

402 403 404
                if view is None:
                    retcode += 1
                    continue
405

406
                dataset_hash = hashDataset(db_name, protocol_name, set_name)
407
                try:
408 409 410
                    view.index(
                        os.path.join(configuration.cache, toPath(dataset_hash, ".db"))
                    )
411
                except RuntimeError as error:
412 413 414 415
                    logger.error("Failed to load the database `%s':", database_name)
                    logger.error("  * Runtime error %s", error)
                    retcode += 1
                    continue
416

417
            else:
418 419 420 421 422 423 424 425 426 427 428 429
                databases_container = start_db_container(
                    configuration,
                    CMD_DB_INDEX,
                    host,
                    db_name,
                    protocol_name,
                    set_name,
                    database,
                    db_set,
                    uid=uid,
                    db_root=db_root,
                )
430
                status = host.wait(databases_container)
431 432 433
                logs = host.logs(databases_container)
                host.rm(databases_container)

434
                if status != 0:
435
                    logger.error("Error occurred: %s", logs)
436
                    retcode += 1
André Anjos's avatar
André Anjos committed
437

438
    return retcode
André Anjos's avatar
André Anjos committed
439 440


Samuel GAIST's avatar
Samuel GAIST committed
441
# ----------------------------------------------------------
André Anjos's avatar
André Anjos committed
442 443


444
def list_index_files(configuration, names):
445

446
    names = common.make_up_local_list(configuration.path, "database", names)
447

448
    retcode = 0
449

450 451
    for database_name in names:
        logger.info("Listing database %s indexes...", database_name)
452

453
        (db_name, database, sets) = load_database_sets(configuration, database_name)
454 455 456
        if database is None:
            retcode += 1
            continue
André Anjos's avatar
André Anjos committed
457

458
        for protocol_name, set_name, db_set in sets:
459 460 461
            dataset_hash = hashDataset(db_name, protocol_name, set_name)
            index_filename = toPath(dataset_hash)
            basename = os.path.splitext(index_filename)[0]
462
            for g in glob.glob(basename + ".*"):
463
                logger.info(g)
464

465
    return retcode
466 467


Samuel GAIST's avatar
Samuel GAIST committed
468
# ----------------------------------------------------------
469 470


471
def delete_index_files(configuration, names):
472

473
    names = common.make_up_local_list(configuration.path, "database", names)
474

475
    retcode = 0
476

477 478
    for database_name in names:
        logger.info("Deleting database %s indexes...", database_name)
479

480
        (db_name, database, sets) = load_database_sets(configuration, database_name)
481 482 483
        if database is None:
            retcode += 1
            continue
484

485
        for protocol_name, set_name, db_set in sets:
486
            for output_name in db_set["outputs"].keys():
487 488
                dataset_hash = hashDataset(db_name, protocol_name, set_name)
                index_filename = toPath(dataset_hash)
489 490 491
                basename = os.path.join(
                    configuration.cache, os.path.splitext(index_filename)[0]
                )
492

493
                for g in glob.glob(basename + ".*"):
494 495
                    logger.info("removing `%s'...", g)
                    os.unlink(g)
496

497 498 499
                common.recursive_rmdir_if_empty(
                    os.path.dirname(basename), configuration.cache
                )
500

501
    return retcode
502 503


Samuel GAIST's avatar
Samuel GAIST committed
504
# ----------------------------------------------------------
505

André Anjos's avatar
André Anjos committed
506

507 508 509 510 511 512 513 514
def view_outputs(
    configuration,
    dataset_name,
    excluded_outputs=None,
    uid=None,
    db_root=None,
    docker=False,
):
515 516
    def data_to_json(data, indent):
        value = common.stringify(data.as_dict())
André Anjos's avatar
André Anjos committed
517

518 519 520 521 522 523 524 525
        value = (
            simplejson.dumps(value, indent=4, cls=NumpyJSONEncoder)
            .replace('"BEAT_LIST_DELIMITER[', "[")
            .replace(']BEAT_LIST_DELIMITER"', "]")
            .replace('"...",', "...")
            .replace('"BEAT_LIST_SIZE(', "(")
            .replace(')BEAT_LIST_SIZE"', ")")
        )
André Anjos's avatar
André Anjos committed
526

527
        return ("\n" + " " * indent).join(value.split("\n"))
André Anjos's avatar
André Anjos committed
528

529 530 531 532
    # Load the infos about the database set
    (db_name, database, sets) = load_database_sets(configuration, dataset_name)
    if (database is None) or (len(sets) != 1):
        return 1
André Anjos's avatar
André Anjos committed
533

534 535 536
    (protocol_name, set_name, db_set) = sets[0]

    if excluded_outputs is not None:
537
        excluded_outputs = map(lambda x: x.strip(), excluded_outputs.split(","))
André Anjos's avatar
André Anjos committed
538

539 540
    # Setup the view so the outputs can be used
    if not docker:
541
        view = database.view(protocol_name, set_name)
542

543 544
        if view is None:
            return 1
545 546

        dataset_hash = hashDataset(db_name, protocol_name, set_name)
547 548 549
        view.setup(
            os.path.join(configuration.cache, toPath(dataset_hash, ".db")), pack=False
        )
550 551
        input_group = inputs.InputGroup(set_name, restricted_access=False)

552 553
        for output_name, dataformat_name in db_set["outputs"].items():
            if excluded_outputs is not None and output_name in excluded_outputs:
554 555
                continue

556 557 558 559 560
            input = inputs.Input(
                output_name,
                database.dataformats[dataformat_name],
                view.data_sources[output_name],
            )
561 562
            input_group.add(input)

563 564 565
    else:
        host = dock.Host(raise_on_errors=False)

566 567 568 569 570 571 572 573 574 575 576 577 578
        (databases_container, db_socket, zmq_context, input_list) = start_db_container(
            configuration,
            CMD_VIEW_OUTPUTS,
            host,
            db_name,
            protocol_name,
            set_name,
            database,
            db_set,
            excluded_outputs=excluded_outputs,
            uid=uid,
            db_root=db_root,
        )
579

580
        input_group = input_list.group(set_name)
André Anjos's avatar
André Anjos committed
581

582 583
    retvalue = 0

584 585 586
    # Display the data
    try:
        previous_start = -1
André Anjos's avatar
André Anjos committed
587

588 589
        while input_group.hasMoreData():
            input_group.next()
André Anjos's avatar
André Anjos committed
590

591 592
            start = input_group.data_index
            end = input_group.data_index_end
André Anjos's avatar
André Anjos committed
593

594
            if start != previous_start:
595
                print(80 * "-")
André Anjos's avatar
André Anjos committed
596

597
                print("FROM %d TO %d" % (start, end))
André Anjos's avatar
André Anjos committed
598

599 600 601 602 603
                whole_inputs = [
                    input_
                    for input_ in input_group
                    if input_.data_index == start and input_.data_index_end == end
                ]
André Anjos's avatar
André Anjos committed
604

605
                for input in whole_inputs:
606
                    label = " - " + str(input.name) + ": "
607
                    print(label + data_to_json(input.data, len(label)))
André Anjos's avatar
André Anjos committed
608

609
                previous_start = start
André Anjos's avatar
André Anjos committed
610

611 612 613 614 615 616
            selected_inputs = [
                input_
                for input_ in input_group
                if input_.data_index == input_group.first_data_index
                and (input_.data_index != start or input_.data_index_end != end)
            ]
André Anjos's avatar
André Anjos committed
617

618
            grouped_inputs = {}
Samuel GAIST's avatar
Samuel GAIST committed
619 620 621
            for input_ in selected_inputs:
                key = (input_.data_index, input_.data_index_end)
                if key not in grouped_inputs:
622 623
                    grouped_inputs[key] = []
                grouped_inputs[key].append(input)
André Anjos's avatar
André Anjos committed
624

625
            sorted_keys = sorted(grouped_inputs.keys())
626 627 628

            for key in sorted_keys:
                print
629
                print("  FROM %d TO %d" % key)
630 631

                for input in grouped_inputs[key]:
632
                    label = "   - " + str(input.name) + ": "
633
                    print(label + data_to_json(input.data, len(label)))
André Anjos's avatar
André Anjos committed
634 635

    except Exception as e:
636
        logger.error("Failed to retrieve the next data: %s", e)
637 638 639 640 641 642 643 644 645 646 647
        retvalue = 1

    if docker:
        host.kill(databases_container)
        status = host.wait(databases_container)
        logs = host.logs(databases_container)
        host.rm(databases_container)
        if status != 0:
            logger.error("Docker error: %s", logs)

    return retvalue
André Anjos's avatar
André Anjos committed
648

649

Samuel GAIST's avatar
Samuel GAIST committed
650
# ----------------------------------------------------------
André Anjos's avatar
André Anjos committed
651 652


653 654 655 656 657 658 659 660
class DatabaseCommand(AssetCommand):
    asset_info = AssetInfo(
        asset_type="database",
        diff_fields=["declaration", "code", "description"],
        push_fields=["name", "declaration", "code", "description"],
    )


661
@click.group(cls=AliasedGroup)
662
@click.pass_context
663
def databases(ctx):
664
    """Database commands"""
665

666

667 668 669 670 671 672 673 674 675 676
CMD_LIST = [
    "list",
    "path",
    "edit",
    "check",
    "status",
    "create",
    "version",
    ("rm", "rm_local"),
    "diff",
677
    "push",
678 679
]

680
commands.initialise_asset_commands(databases, CMD_LIST, DatabaseCommand)
681 682


683
@databases.command()
684 685 686 687
@click.argument("db_names", nargs=-1)
@click.option(
    "--force", help="Performs operation regardless of conflicts", is_flag=True
)
688
@click.pass_context
689
@raise_on_error
690
def pull(ctx, db_names, force):
691
    """Downloads the specified databases from the server.
692

693
       $ beat databases pull [<name>]...
694 695 696

    <name>:
        Database name formatted as "<database>/<version>"
697 698
    """
    configuration = ctx.meta["config"]
699 700 701 702
    with common.make_webapi(configuration) as webapi:
        return pull_impl(webapi, configuration.path, db_names, force, 0, {})


703
@databases.command()
704 705 706 707 708 709 710 711 712 713 714 715 716 717
@click.argument("db_names", nargs=-1)
@click.option(
    "--list", help="List index files matching output if they exist", is_flag=True
)
@click.option(
    "--delete",
    help="Delete index files matching output if they "
    "exist (also, recursively deletes empty directories)",
    is_flag=True,
)
@click.option("--checksum", help="Checksums index files", is_flag=True, default=True)
@click.option("--uid", type=click.INT, default=None)
@click.option("--db-root", help="Database root")
@click.option("--docker", is_flag=True)
718
@click.pass_context
719
@raise_on_error
720
def index(ctx, db_names, list, delete, checksum, uid, db_root, docker):
721
    """Indexes all outputs (of all sets) of a database.
722 723 724

    To index the contents of a database

725
        $ beat databases index simple/1
726 727 728

    To index the contents of a protocol on a database

729
        $ beat databases index simple/1/double
730 731 732

    To index the contents of a set in a protocol on a database

733
        $ beat databases index simple/1/double/double
734 735
    """
    configuration = ctx.meta["config"]
736
    code = 1
737
    if list:
738
        code = list_index_files(configuration, db_names)
739
    elif delete:
740
        code = delete_index_files(configuration, db_names)
741
    elif checksum:
742 743 744
        code = index_outputs(
            configuration, db_names, uid=uid, db_root=db_root, docker=docker
        )
745
    return code
746

747

748
@databases.command()
749 750 751 752 753
@click.argument("set_name", nargs=1)
@click.option("--exclude", help="When viewing, excludes this output", default=None)
@click.option("--uid", type=click.INT, default=None)
@click.option("--db-root", help="Database root")
@click.option("--docker", is_flag=True)
754
@click.pass_context
755
@raise_on_error
756
def view(ctx, set_name, exclude, uid, db_root, docker):
757
    """View the data of the specified dataset.
758 759 760

    To view the contents of a specific set

761
    $ beat databases view simple/1/protocol/set
762 763
    """
    configuration = ctx.meta["config"]
764 765
    if exclude is not None:
        return view_outputs(
766 767
            configuration, set_name, exclude, uid=uid, db_root=db_root, docker=docker
        )
768 769 770
    return view_outputs(
        configuration, set_name, uid=uid, db_root=db_root, docker=docker
    )