databases.py 25.3 KB
Newer Older
André Anjos's avatar
André Anjos committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

###############################################################################
#                                                                             #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
# Contact: beat.support@idiap.ch                                              #
#                                                                             #
# This file is part of the beat.cmdline module of the BEAT platform.          #
#                                                                             #
# Commercial License Usage                                                    #
# Licensees holding valid commercial BEAT licenses may use this file in       #
# accordance with the terms contained in a written agreement between you      #
# and Idiap. For further information contact tto@idiap.ch                     #
#                                                                             #
# Alternatively, this file may be used under the terms of the GNU Affero      #
# Public License version 3 as published by the Free Software and appearing    #
# in the file LICENSE.AGPL included in the packaging of this file.            #
# The BEAT platform is distributed in the hope that it will be useful, but    #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
#                                                                             #
# You should have received a copy of the GNU Affero Public License along      #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
#                                                                             #
###############################################################################

import os
29
import sys
30
import click
André Anjos's avatar
André Anjos committed
31
import glob
32
import random
Samuel GAIST's avatar
Samuel GAIST committed
33
import zmq
34

André Anjos's avatar
André Anjos committed
35
import logging
Samuel GAIST's avatar
Samuel GAIST committed
36

André Anjos's avatar
André Anjos committed
37
38
39

import simplejson

Samuel GAIST's avatar
Samuel GAIST committed
40
41
from beat.core.hash import toPath
from beat.core.hash import hashDataset
André Anjos's avatar
André Anjos committed
42
43
from beat.core.utils import NumpyJSONEncoder
from beat.core.database import Database
44
from beat.core.data import load_data_index, RemoteDataSource
45
46
47
from beat.core import dock
from beat.core import inputs
from beat.core import utils
André Anjos's avatar
André Anjos committed
48
49
50

from . import common

Samuel GAIST's avatar
Samuel GAIST committed
51
logger = logging.getLogger(__name__)
André Anjos's avatar
André Anjos committed
52

53
54
55
56
CMD_DB_INDEX = 'index'
CMD_VIEW_OUTPUTS = 'databases_provider'


Samuel GAIST's avatar
Samuel GAIST committed
57
# ----------------------------------------------------------
58
59
60
61
62
63
64


def load_database_sets(configuration, database_name):
    # Process the name of the database
    parts = database_name.split('/')

    if len(parts) == 2:
Samuel GAIST's avatar
Samuel GAIST committed
65
66
67
        db_name = os.path.join(*parts[:2])
        protocol_filter = None
        set_filter = None
68
69

    elif len(parts) == 3:
Samuel GAIST's avatar
Samuel GAIST committed
70
71
72
        db_name = os.path.join(*parts[:2])
        protocol_filter = parts[2]
        set_filter = None
73
74

    elif len(parts) == 4:
Samuel GAIST's avatar
Samuel GAIST committed
75
76
77
        db_name = os.path.join(*parts[:2])
        protocol_filter = parts[2]
        set_filter = parts[3]
78
79

    else:
Samuel GAIST's avatar
Samuel GAIST committed
80
        logger.error("Database specification should have the format "
Samuel GAIST's avatar
Samuel GAIST committed
81
                     "`<database>/<version>/[<protocol>/[<set>]]', the value "
Samuel GAIST's avatar
Samuel GAIST committed
82
                     "you passed (%s) is not valid", database_name)
Samuel GAIST's avatar
Samuel GAIST committed
83
        return (None, None)
84
85
86

    # Load the dataformat
    dataformat_cache = {}
87
    database = Database(configuration.path,
Samuel GAIST's avatar
Samuel GAIST committed
88
                        db_name, dataformat_cache)
89
    if not database.valid:
Samuel GAIST's avatar
Samuel GAIST committed
90
91
92
93
        logger.error("Failed to load the database `%s':", db_name)
        for e in database.errors:
            logger.error('  * %s', e)
        return (None, None, None)
94
95
96
97
98

    # Filter the protocols
    protocols = database.protocol_names

    if protocol_filter is not None:
Samuel GAIST's avatar
Samuel GAIST committed
99
100
101
102
        if protocol_filter not in protocols:
            logger.error("The database `%s' does not have the protocol `%s' - "
                         "choose one of `%s'", db_name, protocol_filter,
                         ', '.join(protocols))
103

Samuel GAIST's avatar
Samuel GAIST committed
104
            return (None, None, None)
105

Samuel GAIST's avatar
Samuel GAIST committed
106
        protocols = [protocol_filter]
107
108
109
110
111

    # Filter the sets
    loaded_sets = []

    for protocol_name in protocols:
Samuel GAIST's avatar
Samuel GAIST committed
112
        sets = database.set_names(protocol_name)
113

Samuel GAIST's avatar
Samuel GAIST committed
114
115
116
        if set_filter is not None:
            if set_filter not in sets:
                logger.error("The database/protocol `%s/%s' does not have the "
Samuel GAIST's avatar
Samuel GAIST committed
117
118
119
                             "set `%s' - choose one of `%s'",
                             db_name, protocol_name, set_filter,
                             ', '.join(sets))
Samuel GAIST's avatar
Samuel GAIST committed
120
                return (None, None, None)
121

Samuel GAIST's avatar
Samuel GAIST committed
122
            sets = [z for z in sets if z == set_filter]
123

Samuel GAIST's avatar
Samuel GAIST committed
124
125
        loaded_sets.extend([(protocol_name, set_name,
                             database.set(protocol_name, set_name))
Samuel GAIST's avatar
Samuel GAIST committed
126
                            for set_name in sets])
127
128
129
130

    return (db_name, database, loaded_sets)


Samuel GAIST's avatar
Samuel GAIST committed
131
# ----------------------------------------------------------
132
133


134
135
def start_db_container(configuration, cmd, host,
                       db_name, protocol_name, set_name, database, db_set,
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
                       excluded_outputs=None, uid=None, db_root=None):

    input_list = inputs.InputList()

    input_group = inputs.InputGroup(set_name, restricted_access=False)
    input_list.add(input_group)

    db_configuration = {
        'inputs': {},
        'channel': set_name,
    }

    if uid is not None:
        db_configuration['datasets_uid'] = uid

    if db_root is not None:
        db_configuration['datasets_root_path'] = db_root

    for output_name, dataformat_name in db_set['outputs'].items():
Samuel GAIST's avatar
Samuel GAIST committed
155
        if excluded_outputs is not None and output_name in excluded_outputs:
156
157
            continue

158
        dataset_hash = hashDataset(db_name, protocol_name, set_name)
159
        db_configuration['inputs'][output_name] = dict(
Samuel GAIST's avatar
Samuel GAIST committed
160
161
162
163
164
165
166
            database=db_name,
            protocol=protocol_name,
            set=set_name,
            output=output_name,
            channel=set_name,
            hash=dataset_hash,
            path=toPath(dataset_hash, '.db')
167
168
169
170
171
172
173
174
175
176
177
        )

    db_tempdir = utils.temporary_directory()

    with open(os.path.join(db_tempdir, 'configuration.json'), 'wb') as f:
        simplejson.dump(db_configuration, f, indent=4)

    tmp_prefix = os.path.join(db_tempdir, 'prefix')
    if not os.path.exists(tmp_prefix):
        os.makedirs(tmp_prefix)

178
    database.export(tmp_prefix)
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194

    if db_root is None:
        json_path = os.path.join(tmp_prefix, 'databases', db_name + '.json')

        with open(json_path, 'r') as f:
            db_data = simplejson.load(f)

        database_path = db_data['root_folder']
        db_data['root_folder'] = os.path.join('/databases', db_name)

        with open(json_path, 'w') as f:
            simplejson.dump(db_data, f, indent=4)

    try:
        db_envkey = host.db2docker([db_name])
    except:
Samuel GAIST's avatar
Samuel GAIST committed
195
        raise RuntimeError("No environment found for the database `%s' "
196
197
198
199
200
201
                           "- available environments are %s" % (
                               db_name,
                               ", ".join(host.db_environments.keys())))

    # Creation of the container
    # Note: we only support one databases image loaded at the same time
202
203
204
205
206
    CONTAINER_PREFIX = '/beat/prefix'
    CONTAINER_CACHE = '/beat/cache'

    database_port = random.randint(51000, 60000)
    if cmd == CMD_VIEW_OUTPUTS:
Samuel GAIST's avatar
Samuel GAIST committed
207
208
209
210
211
212
        db_cmd = [
            cmd,
            '0.0.0.0:{}'.format(database_port),
            CONTAINER_PREFIX,
            CONTAINER_CACHE
        ]
213
    else:
Samuel GAIST's avatar
Samuel GAIST committed
214
215
216
217
218
219
220
221
        db_cmd = [
            cmd,
            CONTAINER_PREFIX,
            CONTAINER_CACHE,
            db_name,
            protocol_name,
            set_name
        ]
222
223

    databases_container = host.create_container(db_envkey, db_cmd)
224
    if cmd == CMD_VIEW_OUTPUTS:
Samuel GAIST's avatar
Samuel GAIST committed
225
226
        databases_container.add_port(
            database_port, database_port, host_address=host.ip)
227
228
    databases_container.add_volume(db_tempdir, '/beat/prefix')
    databases_container.add_volume(configuration.cache, '/beat/cache')
229
230

    # Specify the volumes to mount inside the container
Samuel GAIST's avatar
Samuel GAIST committed
231
    if 'datasets_root_path' not in db_configuration:
Samuel GAIST's avatar
Samuel GAIST committed
232
233
        databases_container.add_volume(
            database_path, os.path.join('/databases', db_name))
234
235
236
237
238
239
240
    else:
        databases_container.add_volume(db_configuration['datasets_root_path'],
                                       db_configuration['datasets_root_path'])

    # Start the container
    host.start(databases_container)

241
    if cmd == CMD_VIEW_OUTPUTS:
Samuel GAIST's avatar
Samuel GAIST committed
242
243
244
245
246
        # Communicate with container
        zmq_context = zmq.Context()
        db_socket = zmq_context.socket(zmq.PAIR)
        db_address = 'tcp://{}:{}'.format(host.ip, database_port)
        db_socket.connect(db_address)
247

Samuel GAIST's avatar
Samuel GAIST committed
248
        for output_name, dataformat_name in db_set['outputs'].items():
Samuel GAIST's avatar
Samuel GAIST committed
249
250
            if excluded_outputs is not None and \
               output_name in excluded_outputs:
Samuel GAIST's avatar
Samuel GAIST committed
251
                continue
252

Samuel GAIST's avatar
Samuel GAIST committed
253
254
255
            data_source = RemoteDataSource()
            data_source.setup(db_socket, output_name,
                              dataformat_name, configuration.path)
256

Samuel GAIST's avatar
Samuel GAIST committed
257
258
259
260
            input_ = inputs.Input(output_name,
                                  database.dataformats[dataformat_name],
                                  data_source)
            input_group.add(input_)
261

Samuel GAIST's avatar
Samuel GAIST committed
262
        return (databases_container, db_socket, zmq_context, input_list)
263
264

    return databases_container
265
266


Samuel GAIST's avatar
Samuel GAIST committed
267
# ----------------------------------------------------------
268
269


270
def pull_impl(webapi, prefix, names, force, indentation, format_cache):
Samuel GAIST's avatar
Samuel GAIST committed
271
    """Copies databases (and required dataformats) from the server.
André Anjos's avatar
André Anjos committed
272

Samuel GAIST's avatar
Samuel GAIST committed
273
    Parameters:
André Anjos's avatar
André Anjos committed
274

Samuel GAIST's avatar
Samuel GAIST committed
275
276
      webapi (object): An instance of our WebAPI class, prepared to access the
        BEAT server of interest
André Anjos's avatar
André Anjos committed
277

Samuel GAIST's avatar
Samuel GAIST committed
278
279
      prefix (str): A string representing the root of the path in which the
        user objects are stored
André Anjos's avatar
André Anjos committed
280

André Anjos's avatar
André Anjos committed
281
282
283
284
285
      names (:py:class:`list`): A list of strings, each representing the unique
        relative path of the objects to retrieve or a list of usernames from
        which to retrieve objects. If the list is empty, then we pull all
        available objects of a given type. If no user is set, then pull all
        public objects of a given type.
André Anjos's avatar
André Anjos committed
286

Samuel GAIST's avatar
Samuel GAIST committed
287
288
      force (bool): If set to ``True``, then overwrites local changes with the
        remotely retrieved copies.
André Anjos's avatar
André Anjos committed
289

Samuel GAIST's avatar
Samuel GAIST committed
290
291
292
      indentation (int): The indentation level, useful if this function is
        called recursively while downloading different object types. This is
        normally set to ``0`` (zero).
André Anjos's avatar
André Anjos committed
293

Samuel GAIST's avatar
Samuel GAIST committed
294
295
      format_cache (dict): A dictionary containing all dataformats already
        downloaded.
André Anjos's avatar
André Anjos committed
296
297


Samuel GAIST's avatar
Samuel GAIST committed
298
    Returns:
André Anjos's avatar
André Anjos committed
299

Samuel GAIST's avatar
Samuel GAIST committed
300
301
      int: Indicating the exit status of the command, to be reported back to
        the calling process. This value should be zero if everything works OK,
Samuel GAIST's avatar
Samuel GAIST committed
302
        otherwise, different than zero (POSIX compliance).
André Anjos's avatar
André Anjos committed
303

Samuel GAIST's avatar
Samuel GAIST committed
304
    """
André Anjos's avatar
André Anjos committed
305

306
    from .dataformats import pull_impl as dataformats_pull
André Anjos's avatar
André Anjos committed
307

Samuel GAIST's avatar
Samuel GAIST committed
308
    status, names = common.pull(webapi, prefix, 'database', names,
Samuel GAIST's avatar
Samuel GAIST committed
309
310
                                ['declaration', 'code', 'description'],
                                force, indentation)
André Anjos's avatar
André Anjos committed
311

Samuel GAIST's avatar
Samuel GAIST committed
312
313
314
315
316
    # see what dataformats one needs to pull
    dataformats = []
    for name in names:
        obj = Database(prefix, name)
        dataformats.extend(obj.dataformats.keys())
André Anjos's avatar
André Anjos committed
317

Samuel GAIST's avatar
Samuel GAIST committed
318
319
320
    # downloads any formats to which we depend on
    df_status = dataformats_pull(webapi, prefix, dataformats, force,
                                 indentation + 2, format_cache)
André Anjos's avatar
André Anjos committed
321

Samuel GAIST's avatar
Samuel GAIST committed
322
    return status + df_status
André Anjos's avatar
André Anjos committed
323
324


Samuel GAIST's avatar
Samuel GAIST committed
325
# ----------------------------------------------------------
André Anjos's avatar
André Anjos committed
326
327


328
def index_outputs(configuration, names, uid=None, db_root=None, docker=False):
André Anjos's avatar
André Anjos committed
329

330
331
    names = common.make_up_local_list(configuration.path, 'database', names)
    retcode = 0
André Anjos's avatar
André Anjos committed
332

Philip ABBET's avatar
Philip ABBET committed
333
    if docker:
Samuel GAIST's avatar
Samuel GAIST committed
334
        host = dock.Host(raise_on_errors=False)
André Anjos's avatar
André Anjos committed
335

336
    for database_name in names:
Samuel GAIST's avatar
Samuel GAIST committed
337
        logger.info("Indexing database %s...", database_name)
André Anjos's avatar
André Anjos committed
338

Samuel GAIST's avatar
Samuel GAIST committed
339
340
341
342
343
        (db_name, database, sets) = load_database_sets(
            configuration, database_name)
        if database is None:
            retcode += 1
            continue
André Anjos's avatar
André Anjos committed
344

Samuel GAIST's avatar
Samuel GAIST committed
345
346
        for protocol_name, set_name, db_set in sets:
            if not docker:
347
348
349
350
351
352
353
                try:
                    view = database.view(protocol_name, set_name)
                except SyntaxError as error:
                    logger.error("Failed to load the database `%s':",
                                 database_name)
                    logger.error('  * Syntax error: %s', error)
                    view = None
André Anjos's avatar
André Anjos committed
354

Samuel GAIST's avatar
Samuel GAIST committed
355
356
357
                if view is None:
                    retcode += 1
                    continue
358

Samuel GAIST's avatar
Samuel GAIST committed
359
                dataset_hash = hashDataset(db_name, protocol_name, set_name)
360
361
362
363
364
365
366
367
368
                try:
                    view.index(os.path.join(configuration.cache,
                                            toPath(dataset_hash, '.db')))
                except RuntimeError as error:
                  logger.error("Failed to load the database `%s':",
                               database_name)
                  logger.error('  * Runtime error %s', error)
                  retcode += 1
                  continue
369

Samuel GAIST's avatar
Samuel GAIST committed
370
371
372
            else:
                databases_container = \
                    start_db_container(configuration, CMD_DB_INDEX,
Samuel GAIST's avatar
Samuel GAIST committed
373
374
                                       host, db_name, protocol_name, set_name,
                                       database, db_set,
Samuel GAIST's avatar
Samuel GAIST committed
375
376
377
378
379
                                       uid=uid, db_root=db_root
                                       )
                status = host.wait(databases_container)
                if status != 0:
                    retcode += 1
André Anjos's avatar
André Anjos committed
380

381
    return retcode
André Anjos's avatar
André Anjos committed
382
383


Samuel GAIST's avatar
Samuel GAIST committed
384
# ----------------------------------------------------------
André Anjos's avatar
André Anjos committed
385
386


387
def list_index_files(configuration, names):
388

389
    names = common.make_up_local_list(configuration.path, 'database', names)
390

391
    retcode = 0
392

393
394
    for database_name in names:
        logger.info("Listing database %s indexes...", database_name)
395

Samuel GAIST's avatar
Samuel GAIST committed
396
397
        (db_name, database, sets) = load_database_sets(
            configuration, database_name)
398
399
400
        if database is None:
            retcode += 1
            continue
André Anjos's avatar
André Anjos committed
401

402
        for protocol_name, set_name, db_set in sets:
403
404
405
406
407
            dataset_hash = hashDataset(db_name, protocol_name, set_name)
            index_filename = toPath(dataset_hash)
            basename = os.path.splitext(index_filename)[0]
            for g in glob.glob(basename + '.*'):
                logger.info(g)
408

409
    return retcode
410
411


Samuel GAIST's avatar
Samuel GAIST committed
412
# ----------------------------------------------------------
413
414


415
def delete_index_files(configuration, names):
416

417
    names = common.make_up_local_list(configuration.path, 'database', names)
418

419
    retcode = 0
420

421
422
    for database_name in names:
        logger.info("Deleting database %s indexes...", database_name)
423

Samuel GAIST's avatar
Samuel GAIST committed
424
425
        (db_name, database, sets) = load_database_sets(
            configuration, database_name)
426
427
428
        if database is None:
            retcode += 1
            continue
429

430
431
        for protocol_name, set_name, db_set in sets:
            for output_name in db_set['outputs'].keys():
432
433
434
435
                dataset_hash = hashDataset(db_name, protocol_name, set_name)
                index_filename = toPath(dataset_hash)
                basename = os.path.join(configuration.cache,
                                        os.path.splitext(index_filename)[0])
436

437
438
439
                for g in glob.glob(basename + '.*'):
                    logger.info("removing `%s'...", g)
                    os.unlink(g)
440

441
442
                common.recursive_rmdir_if_empty(os.path.dirname(basename),
                                                configuration.cache)
443

444
    return retcode
445
446


Samuel GAIST's avatar
Samuel GAIST committed
447
# ----------------------------------------------------------
448

André Anjos's avatar
André Anjos committed
449

450
451
def view_outputs(configuration, dataset_name, excluded_outputs=None, uid=None,
                 db_root=None, docker=False):
André Anjos's avatar
André Anjos committed
452

453
454
    def data_to_json(data, indent):
        value = common.stringify(data.as_dict())
André Anjos's avatar
André Anjos committed
455

456
457
458
459
460
461
        value = simplejson.dumps(value, indent=4, cls=NumpyJSONEncoder) \
            .replace('"BEAT_LIST_DELIMITER[', '[') \
            .replace(']BEAT_LIST_DELIMITER"', ']') \
            .replace('"...",', '...') \
            .replace('"BEAT_LIST_SIZE(', '(') \
            .replace(')BEAT_LIST_SIZE"', ')')
André Anjos's avatar
André Anjos committed
462

463
        return ('\n' + ' ' * indent).join(value.split('\n'))
André Anjos's avatar
André Anjos committed
464

465
466
467
    # Load the infos about the database set
    (db_name, database, sets) = load_database_sets(configuration, dataset_name)
    if (database is None) or (len(sets) != 1):
468
        sys.exit(1)
469
        return 1
André Anjos's avatar
André Anjos committed
470

471
472
473
    (protocol_name, set_name, db_set) = sets[0]

    if excluded_outputs is not None:
Samuel GAIST's avatar
Samuel GAIST committed
474
475
        excluded_outputs = map(lambda x: x.strip(),
                               excluded_outputs.split(','))
André Anjos's avatar
André Anjos committed
476

477
478
    # Setup the view so the outputs can be used
    if not docker:
Samuel GAIST's avatar
Samuel GAIST committed
479
        view = database.view(protocol_name, set_name)
480

481
        if view is None:
482
            sys.exit(1)
483
            return 1
484
485
486
487
488
489
490

        dataset_hash = hashDataset(db_name, protocol_name, set_name)
        view.setup(os.path.join(configuration.cache,
                                toPath(dataset_hash, '.db')), pack=False)
        input_group = inputs.InputGroup(set_name, restricted_access=False)

        for output_name, dataformat_name in db_set['outputs'].items():
Samuel GAIST's avatar
Samuel GAIST committed
491
492
            if excluded_outputs is not None and \
               output_name in excluded_outputs:
493
494
                continue

Samuel GAIST's avatar
Samuel GAIST committed
495
496
497
            input = inputs.Input(output_name,
                                 database.dataformats[dataformat_name],
                                 view.data_sources[output_name])
498
499
            input_group.add(input)

500
501
502
503
    else:
        host = dock.Host(raise_on_errors=False)

        (databases_container, db_socket, zmq_context, input_list) = \
504
            start_db_container(configuration, CMD_VIEW_OUTPUTS,
Samuel GAIST's avatar
Samuel GAIST committed
505
506
507
508
                               host, db_name, protocol_name,
                               set_name, database, db_set,
                               excluded_outputs=excluded_outputs,
                               uid=uid, db_root=db_root)
509

510
        input_group = input_list.group(set_name)
André Anjos's avatar
André Anjos committed
511

512
513
514
    # Display the data
    try:
        previous_start = -1
André Anjos's avatar
André Anjos committed
515

516
517
        while input_group.hasMoreData():
            input_group.next()
André Anjos's avatar
André Anjos committed
518

519
520
            start = input_group.data_index
            end = input_group.data_index_end
André Anjos's avatar
André Anjos committed
521

522
523
            if start != previous_start:
                print(80 * '-')
André Anjos's avatar
André Anjos committed
524

525
                print('FROM %d TO %d' % (start, end))
André Anjos's avatar
André Anjos committed
526

Samuel GAIST's avatar
Samuel GAIST committed
527
528
529
                whole_inputs = [input_ for input_ in input_group
                                if input_.data_index == start and
                                input_.data_index_end == end]
André Anjos's avatar
André Anjos committed
530

531
532
                for input in whole_inputs:
                    label = ' - ' + str(input.name) + ': '
533
                    print(label + data_to_json(input.data, len(label)))
André Anjos's avatar
André Anjos committed
534

535
                previous_start = start
André Anjos's avatar
André Anjos committed
536

Samuel GAIST's avatar
Samuel GAIST committed
537
538
539
540
541
            selected_inputs = \
                [input_ for input_ in input_group
                 if input_.data_index == input_group.first_data_index and
                 (input_.data_index != start or
                  input_.data_index_end != end)]
André Anjos's avatar
André Anjos committed
542

543
            grouped_inputs = {}
Samuel GAIST's avatar
Samuel GAIST committed
544
545
546
            for input_ in selected_inputs:
                key = (input_.data_index, input_.data_index_end)
                if key not in grouped_inputs:
547
548
                    grouped_inputs[key] = []
                grouped_inputs[key].append(input)
André Anjos's avatar
André Anjos committed
549

550
            sorted_keys = sorted(grouped_inputs.keys())
551
552
553

            for key in sorted_keys:
                print
554
                print('  FROM %d TO %d' % key)
555
556
557

                for input in grouped_inputs[key]:
                    label = '   - ' + str(input.name) + ': '
558
                    print(label + data_to_json(input.data, len(label)))
André Anjos's avatar
André Anjos committed
559
560

    except Exception as e:
561
        logger.error("Failed to retrieve the next data: %s", e)
562
        sys.exit(1)
563
564
        return 1

565
    sys.exit(0)
566
    return 0
André Anjos's avatar
André Anjos committed
567

568

Samuel GAIST's avatar
Samuel GAIST committed
569
# ----------------------------------------------------------
André Anjos's avatar
André Anjos committed
570
571


572
573
@click.group()
@click.pass_context
574
def databases(ctx):
575
576
    """Database commands"""
    pass
577

Samuel GAIST's avatar
Samuel GAIST committed
578

579
@databases.command()
580
581
582
583
584
@click.option('--remote', help='Only acts on the remote copy of the database.',
              is_flag=True)
@click.pass_context
def list(ctx, remote):
    '''Lists all the databases available on the platform.
585

586
    To list all existing databases on your local prefix:
587

588
        $ beat databases list
589
590
591
    '''
    configuration = ctx.meta['config']
    if remote:
592
        with common.make_webapi(configuration) as webapi:
593
594
595
596
597
            return common.display_remote_list(webapi, 'database')
    else:
        return common.display_local_list(configuration.path, 'database')


598
@databases.command()
599
600
601
602
603
@click.argument('db_names', nargs=-1)
@click.pass_context
def check(ctx, db_names):
    '''Checks a local database for validity.

604
    $ beat databases check [<name>]...
605
606
607
608
609
610
611

    <name>:
        Database name formatted as "<database>/<version>"
    '''
    return common.check(ctx.meta['config'].path, 'database', db_names)


612
@databases.command()
613
@click.argument('db_names', nargs=-1)
614
@click.option('--force', help='Performs operation regardless of conflicts',
615
616
617
618
619
              is_flag=True)
@click.pass_context
def pull(ctx, db_names, force):
    '''Downloads the specified databases from the server.

620
       $ beat databases pull [<name>]...
621
622
623
624
625
626
627
628
629

    <name>:
        Database name formatted as "<database>/<version>"
    '''
    configuration = ctx.meta['config']
    with common.make_webapi(configuration) as webapi:
        return pull_impl(webapi, configuration.path, db_names, force, 0, {})


630
@databases.command()
631
@click.argument('db_names', nargs=-1)
632
@click.option('--force', help='Performs operation regardless of conflicts',
633
634
635
636
637
638
639
              is_flag=True)
@click.option('--dry-run', help='Dry run',
              is_flag=True)
@click.pass_context
def push(ctx, db_names, force, dry_run):
    '''Uploads databases to the server (must provide a valid admin token).

640
    $ beat databases push [<name>]...
641
642
643
644
645
646
647
648
649
650
651
652
653

    <name>:
        Database name formatted as "<database>/<version>"

    '''
    configuration = ctx.meta['config']
    with common.make_webapi(configuration) as webapi:
        return common.push(webapi, configuration.path, 'database',
                           db_names, ['name', 'declaration',
                                      'code', 'description'],
                           {}, force, dry_run, 0)


654
@databases.command()
655
656
657
658
659
@click.argument('db_names', nargs=-1)
@click.pass_context
def diff(ctx, db_names):
    '''Shows changes between the local database and the remote version.

660
    $ beat databases diff [<name>]...
661
662
663
664
665
666
667
668
669
670
671
672
673

    <name>:
        Database name formatted as "<database>/<version>"
    '''
    configuration = ctx.meta['config']
    if len(db_names) < 1:
        raise click.ClickException("Requires at least one database name")
    with common.make_webapi(configuration) as webapi:
        return common.diff(webapi, configuration.path, 'database',
                           db_names[0],
                           ['declaration', 'code', 'description'])


674
@databases.command()
675
676
677
678
679
680
681
682
@click.pass_context
def status(ctx):
    '''Shows (editing) status for all available databases'''
    configuration = ctx.meta['config']
    with common.make_webapi(configuration) as webapi:
        return common.status(webapi, configuration.path, 'database')[0]


683
@databases.command()
684
685
686
687
688
@click.argument('db_names', nargs=-1)
@click.pass_context
def version(ctx, db_names):
    '''Creates a new version of an existing database.

689
    $ beat databases version [<name>]...
690
691
692
693
694
695
696
697
698
699
700

    <name>:
        Database name formatted as "<database>/<version>"

    '''
    configuration = ctx.meta['config']
    if len(db_names) < 1:
        raise click.ClickException("Requires at least one database name")
    return common.new_version(configuration.path, 'database', db_names[0])


701
@databases.command()
702
703
704
705
706
707
@click.argument('db_names', nargs=-1)
@click.option('--list', help='List index files matching output if they exist',
              is_flag=True)
@click.option('--delete', help='Delete index files matching output if they '
              'exist (also, recursively deletes empty directories)',
              is_flag=True)
708
709
@click.option('--checksum', help='Checksums index files', is_flag=True,
              default=True)
710
711
712
713
714
715
716
717
718
@click.option('--uid', type=click.INT, default=None)
@click.option('--db-root', help="Database root")
@click.option('--docker', is_flag=True)
@click.pass_context
def index(ctx, db_names, list, delete, checksum, uid, db_root, docker):
    '''Indexes all outputs (of all sets) of a database.

    To index the contents of a database

719
        $ beat databases index simple/1
720
721
722

    To index the contents of a protocol on a database

723
        $ beat databases index simple/1/double
724
725
726

    To index the contents of a set in a protocol on a database

727
        $ beat databases index simple/1/double/double
728
729
    '''
    configuration = ctx.meta['config']
730
    code = 1
731
    if list:
732
        code = list_index_files(configuration, db_names)
733
    elif delete:
734
        code = delete_index_files(configuration, db_names)
735
    elif checksum:
736
737
738
739
        code = index_outputs(configuration, db_names, uid=uid,
                             db_root=db_root, docker=docker)
    sys.exit(code)
    return code
Samuel GAIST's avatar
Samuel GAIST committed
740

741
742
@databases.command()
@click.argument('set_name', nargs=1)
743
744
745
746
747
748
749
750
751
752
753
@click.option('--exclude', help='When viewing, excludes this output',
              default=None)
@click.option('--uid', type=click.INT, default=None)
@click.option('--db-root', help="Database root")
@click.option('--docker', is_flag=True)
@click.pass_context
def view(ctx, set_name, exclude, uid, db_root, docker):
    '''View the data of the specified dataset.

    To view the contents of a specific set

754
    $ beat databases view simple/1/protocol/set
755
756
757
758
759
760
761
762
763
    '''
    configuration = ctx.meta['config']
    if exclude is not None:
        return view_outputs(
            configuration, set_name, exclude, uid=uid, db_root=db_root,
            docker=docker)
    return view_outputs(
        configuration, set_name, uid=uid, db_root=db_root, docker=docker
    )