databases.py 25.4 KB
Newer Older
André Anjos's avatar
André Anjos committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

###############################################################################
#                                                                             #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
# Contact: beat.support@idiap.ch                                              #
#                                                                             #
# This file is part of the beat.cmdline module of the BEAT platform.          #
#                                                                             #
# Commercial License Usage                                                    #
# Licensees holding valid commercial BEAT licenses may use this file in       #
# accordance with the terms contained in a written agreement between you      #
# and Idiap. For further information contact tto@idiap.ch                     #
#                                                                             #
# Alternatively, this file may be used under the terms of the GNU Affero      #
# Public License version 3 as published by the Free Software and appearing    #
# in the file LICENSE.AGPL included in the packaging of this file.            #
# The BEAT platform is distributed in the hope that it will be useful, but    #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
#                                                                             #
# You should have received a copy of the GNU Affero Public License along      #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
#                                                                             #
###############################################################################

import os
29
import sys
30
import click
André Anjos's avatar
André Anjos committed
31
import glob
32
import random
Samuel GAIST's avatar
Samuel GAIST committed
33
import zmq
34

André Anjos's avatar
André Anjos committed
35
import logging
Samuel GAIST's avatar
Samuel GAIST committed
36

André Anjos's avatar
André Anjos committed
37 38 39

import simplejson

40
from beat.cmdline.scripts.click_helper import AliasedGroup
Samuel GAIST's avatar
Samuel GAIST committed
41 42
from beat.core.hash import toPath
from beat.core.hash import hashDataset
André Anjos's avatar
André Anjos committed
43 44
from beat.core.utils import NumpyJSONEncoder
from beat.core.database import Database
45
from beat.core.data import load_data_index, RemoteDataSource
46 47 48
from beat.core import dock
from beat.core import inputs
from beat.core import utils
André Anjos's avatar
André Anjos committed
49 50 51

from . import common

Samuel GAIST's avatar
Samuel GAIST committed
52
logger = logging.getLogger(__name__)
André Anjos's avatar
André Anjos committed
53

54 55 56 57
CMD_DB_INDEX = 'index'
CMD_VIEW_OUTPUTS = 'databases_provider'


Samuel GAIST's avatar
Samuel GAIST committed
58
# ----------------------------------------------------------
59 60 61 62 63 64 65


def load_database_sets(configuration, database_name):
    # Process the name of the database
    parts = database_name.split('/')

    if len(parts) == 2:
66 67 68
        db_name = os.path.join(*parts[:2])
        protocol_filter = None
        set_filter = None
69 70

    elif len(parts) == 3:
71 72 73
        db_name = os.path.join(*parts[:2])
        protocol_filter = parts[2]
        set_filter = None
74 75

    elif len(parts) == 4:
76 77 78
        db_name = os.path.join(*parts[:2])
        protocol_filter = parts[2]
        set_filter = parts[3]
79 80

    else:
81
        logger.error("Database specification should have the format "
Samuel GAIST's avatar
Samuel GAIST committed
82
                     "`<database>/<version>/[<protocol>/[<set>]]', the value "
Samuel GAIST's avatar
Samuel GAIST committed
83
                     "you passed (%s) is not valid", database_name)
84
        return (None, None)
85 86 87

    # Load the dataformat
    dataformat_cache = {}
88
    database = Database(configuration.path,
89
                        db_name, dataformat_cache)
90
    if not database.valid:
91 92 93 94
        logger.error("Failed to load the database `%s':", db_name)
        for e in database.errors:
            logger.error('  * %s', e)
        return (None, None, None)
95 96 97 98 99

    # Filter the protocols
    protocols = database.protocol_names

    if protocol_filter is not None:
100 101 102 103
        if protocol_filter not in protocols:
            logger.error("The database `%s' does not have the protocol `%s' - "
                         "choose one of `%s'", db_name, protocol_filter,
                         ', '.join(protocols))
104

105
            return (None, None, None)
106

107
        protocols = [protocol_filter]
108 109 110 111 112

    # Filter the sets
    loaded_sets = []

    for protocol_name in protocols:
113
        sets = database.set_names(protocol_name)
114

115 116 117
        if set_filter is not None:
            if set_filter not in sets:
                logger.error("The database/protocol `%s/%s' does not have the "
Samuel GAIST's avatar
Samuel GAIST committed
118 119 120
                             "set `%s' - choose one of `%s'",
                             db_name, protocol_name, set_filter,
                             ', '.join(sets))
121
                return (None, None, None)
122

123
            sets = [z for z in sets if z == set_filter]
124

Samuel GAIST's avatar
Samuel GAIST committed
125 126
        loaded_sets.extend([(protocol_name, set_name,
                             database.set(protocol_name, set_name))
127
                            for set_name in sets])
128 129 130 131

    return (db_name, database, loaded_sets)


Samuel GAIST's avatar
Samuel GAIST committed
132
# ----------------------------------------------------------
133 134


135 136
def start_db_container(configuration, cmd, host,
                       db_name, protocol_name, set_name, database, db_set,
137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
                       excluded_outputs=None, uid=None, db_root=None):

    input_list = inputs.InputList()

    input_group = inputs.InputGroup(set_name, restricted_access=False)
    input_list.add(input_group)

    db_configuration = {
        'inputs': {},
        'channel': set_name,
    }

    if uid is not None:
        db_configuration['datasets_uid'] = uid

    if db_root is not None:
        db_configuration['datasets_root_path'] = db_root

    for output_name, dataformat_name in db_set['outputs'].items():
Samuel GAIST's avatar
Samuel GAIST committed
156
        if excluded_outputs is not None and output_name in excluded_outputs:
157 158
            continue

159
        dataset_hash = hashDataset(db_name, protocol_name, set_name)
160
        db_configuration['inputs'][output_name] = dict(
161 162 163 164 165 166 167
            database=db_name,
            protocol=protocol_name,
            set=set_name,
            output=output_name,
            channel=set_name,
            hash=dataset_hash,
            path=toPath(dataset_hash, '.db')
168 169 170 171 172 173 174 175 176 177 178
        )

    db_tempdir = utils.temporary_directory()

    with open(os.path.join(db_tempdir, 'configuration.json'), 'wb') as f:
        simplejson.dump(db_configuration, f, indent=4)

    tmp_prefix = os.path.join(db_tempdir, 'prefix')
    if not os.path.exists(tmp_prefix):
        os.makedirs(tmp_prefix)

179
    database.export(tmp_prefix)
180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195

    if db_root is None:
        json_path = os.path.join(tmp_prefix, 'databases', db_name + '.json')

        with open(json_path, 'r') as f:
            db_data = simplejson.load(f)

        database_path = db_data['root_folder']
        db_data['root_folder'] = os.path.join('/databases', db_name)

        with open(json_path, 'w') as f:
            simplejson.dump(db_data, f, indent=4)

    try:
        db_envkey = host.db2docker([db_name])
    except:
196
        raise RuntimeError("No environment found for the database `%s' "
197 198 199 200 201 202
                           "- available environments are %s" % (
                               db_name,
                               ", ".join(host.db_environments.keys())))

    # Creation of the container
    # Note: we only support one databases image loaded at the same time
203 204 205 206 207
    CONTAINER_PREFIX = '/beat/prefix'
    CONTAINER_CACHE = '/beat/cache'

    database_port = random.randint(51000, 60000)
    if cmd == CMD_VIEW_OUTPUTS:
208 209 210 211 212 213
        db_cmd = [
            cmd,
            '0.0.0.0:{}'.format(database_port),
            CONTAINER_PREFIX,
            CONTAINER_CACHE
        ]
214
    else:
215 216 217 218 219 220 221 222
        db_cmd = [
            cmd,
            CONTAINER_PREFIX,
            CONTAINER_CACHE,
            db_name,
            protocol_name,
            set_name
        ]
223 224

    databases_container = host.create_container(db_envkey, db_cmd)
225
    if cmd == CMD_VIEW_OUTPUTS:
226 227
        databases_container.add_port(
            database_port, database_port, host_address=host.ip)
228 229
    databases_container.add_volume(db_tempdir, '/beat/prefix')
    databases_container.add_volume(configuration.cache, '/beat/cache')
230 231

    # Specify the volumes to mount inside the container
Samuel GAIST's avatar
Samuel GAIST committed
232
    if 'datasets_root_path' not in db_configuration:
233 234
        databases_container.add_volume(
            database_path, os.path.join('/databases', db_name))
235 236 237 238 239 240 241
    else:
        databases_container.add_volume(db_configuration['datasets_root_path'],
                                       db_configuration['datasets_root_path'])

    # Start the container
    host.start(databases_container)

242
    if cmd == CMD_VIEW_OUTPUTS:
243 244 245 246 247
        # Communicate with container
        zmq_context = zmq.Context()
        db_socket = zmq_context.socket(zmq.PAIR)
        db_address = 'tcp://{}:{}'.format(host.ip, database_port)
        db_socket.connect(db_address)
248

249
        for output_name, dataformat_name in db_set['outputs'].items():
Samuel GAIST's avatar
Samuel GAIST committed
250 251
            if excluded_outputs is not None and \
               output_name in excluded_outputs:
252
                continue
253

254 255 256
            data_source = RemoteDataSource()
            data_source.setup(db_socket, output_name,
                              dataformat_name, configuration.path)
257

Samuel GAIST's avatar
Samuel GAIST committed
258 259 260 261
            input_ = inputs.Input(output_name,
                                  database.dataformats[dataformat_name],
                                  data_source)
            input_group.add(input_)
262

263
        return (databases_container, db_socket, zmq_context, input_list)
264 265

    return databases_container
266 267


Samuel GAIST's avatar
Samuel GAIST committed
268
# ----------------------------------------------------------
269 270


271
def pull_impl(webapi, prefix, names, force, indentation, format_cache):
272
    """Copies databases (and required dataformats) from the server.
André Anjos's avatar
André Anjos committed
273

274
    Parameters:
André Anjos's avatar
André Anjos committed
275

276 277
      webapi (object): An instance of our WebAPI class, prepared to access the
        BEAT server of interest
André Anjos's avatar
André Anjos committed
278

Samuel GAIST's avatar
Samuel GAIST committed
279 280
      prefix (str): A string representing the root of the path in which the
        user objects are stored
André Anjos's avatar
André Anjos committed
281

André Anjos's avatar
André Anjos committed
282 283 284 285 286
      names (:py:class:`list`): A list of strings, each representing the unique
        relative path of the objects to retrieve or a list of usernames from
        which to retrieve objects. If the list is empty, then we pull all
        available objects of a given type. If no user is set, then pull all
        public objects of a given type.
André Anjos's avatar
André Anjos committed
287

288 289
      force (bool): If set to ``True``, then overwrites local changes with the
        remotely retrieved copies.
André Anjos's avatar
André Anjos committed
290

Samuel GAIST's avatar
Samuel GAIST committed
291 292 293
      indentation (int): The indentation level, useful if this function is
        called recursively while downloading different object types. This is
        normally set to ``0`` (zero).
André Anjos's avatar
André Anjos committed
294

295 296
      format_cache (dict): A dictionary containing all dataformats already
        downloaded.
André Anjos's avatar
André Anjos committed
297 298


299
    Returns:
André Anjos's avatar
André Anjos committed
300

Samuel GAIST's avatar
Samuel GAIST committed
301 302
      int: Indicating the exit status of the command, to be reported back to
        the calling process. This value should be zero if everything works OK,
303
        otherwise, different than zero (POSIX compliance).
André Anjos's avatar
André Anjos committed
304

305
    """
André Anjos's avatar
André Anjos committed
306

307
    from .dataformats import pull_impl as dataformats_pull
André Anjos's avatar
André Anjos committed
308

309
    status, names = common.pull(webapi, prefix, 'database', names,
Samuel GAIST's avatar
Samuel GAIST committed
310 311
                                ['declaration', 'code', 'description'],
                                force, indentation)
André Anjos's avatar
André Anjos committed
312

313 314 315 316 317
    # see what dataformats one needs to pull
    dataformats = []
    for name in names:
        obj = Database(prefix, name)
        dataformats.extend(obj.dataformats.keys())
André Anjos's avatar
André Anjos committed
318

319 320 321
    # downloads any formats to which we depend on
    df_status = dataformats_pull(webapi, prefix, dataformats, force,
                                 indentation + 2, format_cache)
André Anjos's avatar
André Anjos committed
322

323
    return status + df_status
André Anjos's avatar
André Anjos committed
324 325


Samuel GAIST's avatar
Samuel GAIST committed
326
# ----------------------------------------------------------
André Anjos's avatar
André Anjos committed
327 328


329
def index_outputs(configuration, names, uid=None, db_root=None, docker=False):
André Anjos's avatar
André Anjos committed
330

331 332
    names = common.make_up_local_list(configuration.path, 'database', names)
    retcode = 0
André Anjos's avatar
André Anjos committed
333

Philip ABBET's avatar
Philip ABBET committed
334
    if docker:
335
        host = dock.Host(raise_on_errors=False)
André Anjos's avatar
André Anjos committed
336

337
    for database_name in names:
338
        logger.info("Indexing database %s...", database_name)
André Anjos's avatar
André Anjos committed
339

340 341 342 343 344
        (db_name, database, sets) = load_database_sets(
            configuration, database_name)
        if database is None:
            retcode += 1
            continue
André Anjos's avatar
André Anjos committed
345

346 347
        for protocol_name, set_name, db_set in sets:
            if not docker:
348 349 350 351 352 353 354
                try:
                    view = database.view(protocol_name, set_name)
                except SyntaxError as error:
                    logger.error("Failed to load the database `%s':",
                                 database_name)
                    logger.error('  * Syntax error: %s', error)
                    view = None
André Anjos's avatar
André Anjos committed
355

356 357 358
                if view is None:
                    retcode += 1
                    continue
359

360
                dataset_hash = hashDataset(db_name, protocol_name, set_name)
361 362 363 364 365 366 367 368 369
                try:
                    view.index(os.path.join(configuration.cache,
                                            toPath(dataset_hash, '.db')))
                except RuntimeError as error:
                  logger.error("Failed to load the database `%s':",
                               database_name)
                  logger.error('  * Runtime error %s', error)
                  retcode += 1
                  continue
370

371 372 373
            else:
                databases_container = \
                    start_db_container(configuration, CMD_DB_INDEX,
Samuel GAIST's avatar
Samuel GAIST committed
374 375
                                       host, db_name, protocol_name, set_name,
                                       database, db_set,
376 377 378 379 380
                                       uid=uid, db_root=db_root
                                       )
                status = host.wait(databases_container)
                if status != 0:
                    retcode += 1
André Anjos's avatar
André Anjos committed
381

382
    return retcode
André Anjos's avatar
André Anjos committed
383 384


Samuel GAIST's avatar
Samuel GAIST committed
385
# ----------------------------------------------------------
André Anjos's avatar
André Anjos committed
386 387


388
def list_index_files(configuration, names):
389

390
    names = common.make_up_local_list(configuration.path, 'database', names)
391

392
    retcode = 0
393

394 395
    for database_name in names:
        logger.info("Listing database %s indexes...", database_name)
396

397 398
        (db_name, database, sets) = load_database_sets(
            configuration, database_name)
399 400 401
        if database is None:
            retcode += 1
            continue
André Anjos's avatar
André Anjos committed
402

403
        for protocol_name, set_name, db_set in sets:
404 405 406 407 408
            dataset_hash = hashDataset(db_name, protocol_name, set_name)
            index_filename = toPath(dataset_hash)
            basename = os.path.splitext(index_filename)[0]
            for g in glob.glob(basename + '.*'):
                logger.info(g)
409

410
    return retcode
411 412


Samuel GAIST's avatar
Samuel GAIST committed
413
# ----------------------------------------------------------
414 415


416
def delete_index_files(configuration, names):
417

418
    names = common.make_up_local_list(configuration.path, 'database', names)
419

420
    retcode = 0
421

422 423
    for database_name in names:
        logger.info("Deleting database %s indexes...", database_name)
424

425 426
        (db_name, database, sets) = load_database_sets(
            configuration, database_name)
427 428 429
        if database is None:
            retcode += 1
            continue
430

431 432
        for protocol_name, set_name, db_set in sets:
            for output_name in db_set['outputs'].keys():
433 434 435 436
                dataset_hash = hashDataset(db_name, protocol_name, set_name)
                index_filename = toPath(dataset_hash)
                basename = os.path.join(configuration.cache,
                                        os.path.splitext(index_filename)[0])
437

438 439 440
                for g in glob.glob(basename + '.*'):
                    logger.info("removing `%s'...", g)
                    os.unlink(g)
441

442 443
                common.recursive_rmdir_if_empty(os.path.dirname(basename),
                                                configuration.cache)
444

445
    return retcode
446 447


Samuel GAIST's avatar
Samuel GAIST committed
448
# ----------------------------------------------------------
449

André Anjos's avatar
André Anjos committed
450

451 452
def view_outputs(configuration, dataset_name, excluded_outputs=None, uid=None,
                 db_root=None, docker=False):
André Anjos's avatar
André Anjos committed
453

454 455
    def data_to_json(data, indent):
        value = common.stringify(data.as_dict())
André Anjos's avatar
André Anjos committed
456

457 458 459 460 461 462
        value = simplejson.dumps(value, indent=4, cls=NumpyJSONEncoder) \
            .replace('"BEAT_LIST_DELIMITER[', '[') \
            .replace(']BEAT_LIST_DELIMITER"', ']') \
            .replace('"...",', '...') \
            .replace('"BEAT_LIST_SIZE(', '(') \
            .replace(')BEAT_LIST_SIZE"', ')')
André Anjos's avatar
André Anjos committed
463

464
        return ('\n' + ' ' * indent).join(value.split('\n'))
André Anjos's avatar
André Anjos committed
465

466 467 468
    # Load the infos about the database set
    (db_name, database, sets) = load_database_sets(configuration, dataset_name)
    if (database is None) or (len(sets) != 1):
469
        sys.exit(1)
470
        return 1
André Anjos's avatar
André Anjos committed
471

472 473 474
    (protocol_name, set_name, db_set) = sets[0]

    if excluded_outputs is not None:
475 476
        excluded_outputs = map(lambda x: x.strip(),
                               excluded_outputs.split(','))
André Anjos's avatar
André Anjos committed
477

478 479
    # Setup the view so the outputs can be used
    if not docker:
480
        view = database.view(protocol_name, set_name)
481

482
        if view is None:
483
            sys.exit(1)
484
            return 1
485 486 487 488 489 490 491

        dataset_hash = hashDataset(db_name, protocol_name, set_name)
        view.setup(os.path.join(configuration.cache,
                                toPath(dataset_hash, '.db')), pack=False)
        input_group = inputs.InputGroup(set_name, restricted_access=False)

        for output_name, dataformat_name in db_set['outputs'].items():
Samuel GAIST's avatar
Samuel GAIST committed
492 493
            if excluded_outputs is not None and \
               output_name in excluded_outputs:
494 495
                continue

Samuel GAIST's avatar
Samuel GAIST committed
496 497 498
            input = inputs.Input(output_name,
                                 database.dataformats[dataformat_name],
                                 view.data_sources[output_name])
499 500
            input_group.add(input)

501 502 503 504
    else:
        host = dock.Host(raise_on_errors=False)

        (databases_container, db_socket, zmq_context, input_list) = \
505
            start_db_container(configuration, CMD_VIEW_OUTPUTS,
Samuel GAIST's avatar
Samuel GAIST committed
506 507 508 509
                               host, db_name, protocol_name,
                               set_name, database, db_set,
                               excluded_outputs=excluded_outputs,
                               uid=uid, db_root=db_root)
510

511
        input_group = input_list.group(set_name)
André Anjos's avatar
André Anjos committed
512

513 514 515
    # Display the data
    try:
        previous_start = -1
André Anjos's avatar
André Anjos committed
516

517 518
        while input_group.hasMoreData():
            input_group.next()
André Anjos's avatar
André Anjos committed
519

520 521
            start = input_group.data_index
            end = input_group.data_index_end
André Anjos's avatar
André Anjos committed
522

523 524
            if start != previous_start:
                print(80 * '-')
André Anjos's avatar
André Anjos committed
525

526
                print('FROM %d TO %d' % (start, end))
André Anjos's avatar
André Anjos committed
527

Samuel GAIST's avatar
Samuel GAIST committed
528 529 530
                whole_inputs = [input_ for input_ in input_group
                                if input_.data_index == start and
                                input_.data_index_end == end]
André Anjos's avatar
André Anjos committed
531

532 533
                for input in whole_inputs:
                    label = ' - ' + str(input.name) + ': '
534
                    print(label + data_to_json(input.data, len(label)))
André Anjos's avatar
André Anjos committed
535

536
                previous_start = start
André Anjos's avatar
André Anjos committed
537

Samuel GAIST's avatar
Samuel GAIST committed
538 539 540 541 542
            selected_inputs = \
                [input_ for input_ in input_group
                 if input_.data_index == input_group.first_data_index and
                 (input_.data_index != start or
                  input_.data_index_end != end)]
André Anjos's avatar
André Anjos committed
543

544
            grouped_inputs = {}
Samuel GAIST's avatar
Samuel GAIST committed
545 546 547
            for input_ in selected_inputs:
                key = (input_.data_index, input_.data_index_end)
                if key not in grouped_inputs:
548 549
                    grouped_inputs[key] = []
                grouped_inputs[key].append(input)
André Anjos's avatar
André Anjos committed
550

551
            sorted_keys = sorted(grouped_inputs.keys())
552 553 554

            for key in sorted_keys:
                print
555
                print('  FROM %d TO %d' % key)
556 557 558

                for input in grouped_inputs[key]:
                    label = '   - ' + str(input.name) + ': '
559
                    print(label + data_to_json(input.data, len(label)))
André Anjos's avatar
André Anjos committed
560 561

    except Exception as e:
562
        logger.error("Failed to retrieve the next data: %s", e)
563
        sys.exit(1)
564 565
        return 1

566
    sys.exit(0)
567
    return 0
André Anjos's avatar
André Anjos committed
568

569

Samuel GAIST's avatar
Samuel GAIST committed
570
# ----------------------------------------------------------
André Anjos's avatar
André Anjos committed
571 572


573
@click.group(cls=AliasedGroup)
574
@click.pass_context
575
def databases(ctx):
576 577
    """Database commands"""
    pass
578

579

580
@databases.command()
581 582 583 584 585
@click.option('--remote', help='Only acts on the remote copy of the database.',
              is_flag=True)
@click.pass_context
def list(ctx, remote):
    '''Lists all the databases available on the platform.
586

587
    To list all existing databases on your local prefix:
588

589
        $ beat databases list
590 591 592
    '''
    configuration = ctx.meta['config']
    if remote:
593
        with common.make_webapi(configuration) as webapi:
594 595 596 597 598
            return common.display_remote_list(webapi, 'database')
    else:
        return common.display_local_list(configuration.path, 'database')


599
@databases.command()
600 601 602 603 604
@click.argument('db_names', nargs=-1)
@click.pass_context
def check(ctx, db_names):
    '''Checks a local database for validity.

605
    $ beat databases check [<name>]...
606 607 608 609 610 611 612

    <name>:
        Database name formatted as "<database>/<version>"
    '''
    return common.check(ctx.meta['config'].path, 'database', db_names)


613
@databases.command()
614
@click.argument('db_names', nargs=-1)
615
@click.option('--force', help='Performs operation regardless of conflicts',
616 617 618 619 620
              is_flag=True)
@click.pass_context
def pull(ctx, db_names, force):
    '''Downloads the specified databases from the server.

621
       $ beat databases pull [<name>]...
622 623 624 625 626 627 628 629 630

    <name>:
        Database name formatted as "<database>/<version>"
    '''
    configuration = ctx.meta['config']
    with common.make_webapi(configuration) as webapi:
        return pull_impl(webapi, configuration.path, db_names, force, 0, {})


631
@databases.command()
632
@click.argument('db_names', nargs=-1)
633
@click.option('--force', help='Performs operation regardless of conflicts',
634 635 636 637 638 639 640
              is_flag=True)
@click.option('--dry-run', help='Dry run',
              is_flag=True)
@click.pass_context
def push(ctx, db_names, force, dry_run):
    '''Uploads databases to the server (must provide a valid admin token).

641
    $ beat databases push [<name>]...
642 643 644 645 646 647 648 649 650 651 652 653 654

    <name>:
        Database name formatted as "<database>/<version>"

    '''
    configuration = ctx.meta['config']
    with common.make_webapi(configuration) as webapi:
        return common.push(webapi, configuration.path, 'database',
                           db_names, ['name', 'declaration',
                                      'code', 'description'],
                           {}, force, dry_run, 0)


655
@databases.command()
656 657 658 659 660
@click.argument('db_names', nargs=-1)
@click.pass_context
def diff(ctx, db_names):
    '''Shows changes between the local database and the remote version.

661
    $ beat databases diff [<name>]...
662 663 664 665 666 667 668 669 670 671 672 673 674

    <name>:
        Database name formatted as "<database>/<version>"
    '''
    configuration = ctx.meta['config']
    if len(db_names) < 1:
        raise click.ClickException("Requires at least one database name")
    with common.make_webapi(configuration) as webapi:
        return common.diff(webapi, configuration.path, 'database',
                           db_names[0],
                           ['declaration', 'code', 'description'])


675
@databases.command()
676 677 678 679 680 681 682 683
@click.pass_context
def status(ctx):
    '''Shows (editing) status for all available databases'''
    configuration = ctx.meta['config']
    with common.make_webapi(configuration) as webapi:
        return common.status(webapi, configuration.path, 'database')[0]


684
@databases.command()
685 686 687 688 689
@click.argument('db_names', nargs=-1)
@click.pass_context
def version(ctx, db_names):
    '''Creates a new version of an existing database.

690
    $ beat databases version [<name>]...
691 692 693 694 695 696 697 698 699 700 701

    <name>:
        Database name formatted as "<database>/<version>"

    '''
    configuration = ctx.meta['config']
    if len(db_names) < 1:
        raise click.ClickException("Requires at least one database name")
    return common.new_version(configuration.path, 'database', db_names[0])


702
@databases.command()
703 704 705 706 707 708
@click.argument('db_names', nargs=-1)
@click.option('--list', help='List index files matching output if they exist',
              is_flag=True)
@click.option('--delete', help='Delete index files matching output if they '
              'exist (also, recursively deletes empty directories)',
              is_flag=True)
709 710
@click.option('--checksum', help='Checksums index files', is_flag=True,
              default=True)
711 712 713 714 715 716 717 718 719
@click.option('--uid', type=click.INT, default=None)
@click.option('--db-root', help="Database root")
@click.option('--docker', is_flag=True)
@click.pass_context
def index(ctx, db_names, list, delete, checksum, uid, db_root, docker):
    '''Indexes all outputs (of all sets) of a database.

    To index the contents of a database

720
        $ beat databases index simple/1
721 722 723

    To index the contents of a protocol on a database

724
        $ beat databases index simple/1/double
725 726 727

    To index the contents of a set in a protocol on a database

728
        $ beat databases index simple/1/double/double
729 730
    '''
    configuration = ctx.meta['config']
731
    code = 1
732
    if list:
733
        code = list_index_files(configuration, db_names)
734
    elif delete:
735
        code = delete_index_files(configuration, db_names)
736
    elif checksum:
737 738 739 740
        code = index_outputs(configuration, db_names, uid=uid,
                             db_root=db_root, docker=docker)
    sys.exit(code)
    return code
741

742 743
@databases.command()
@click.argument('set_name', nargs=1)
744 745 746 747 748 749 750 751 752 753 754
@click.option('--exclude', help='When viewing, excludes this output',
              default=None)
@click.option('--uid', type=click.INT, default=None)
@click.option('--db-root', help="Database root")
@click.option('--docker', is_flag=True)
@click.pass_context
def view(ctx, set_name, exclude, uid, db_root, docker):
    '''View the data of the specified dataset.

    To view the contents of a specific set

755
    $ beat databases view simple/1/protocol/set
756 757 758 759 760 761 762 763 764
    '''
    configuration = ctx.meta['config']
    if exclude is not None:
        return view_outputs(
            configuration, set_name, exclude, uid=uid, db_root=db_root,
            docker=docker)
    return view_outputs(
        configuration, set_name, uid=uid, db_root=db_root, docker=docker
    )