databases.py 24.4 KB
Newer Older
André Anjos's avatar
André Anjos committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

###############################################################################
#                                                                             #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
# Contact: beat.support@idiap.ch                                              #
#                                                                             #
# This file is part of the beat.cmdline module of the BEAT platform.          #
#                                                                             #
# Commercial License Usage                                                    #
# Licensees holding valid commercial BEAT licenses may use this file in       #
# accordance with the terms contained in a written agreement between you      #
# and Idiap. For further information contact tto@idiap.ch                     #
#                                                                             #
# Alternatively, this file may be used under the terms of the GNU Affero      #
# Public License version 3 as published by the Free Software and appearing    #
# in the file LICENSE.AGPL included in the packaging of this file.            #
# The BEAT platform is distributed in the hope that it will be useful, but    #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
#                                                                             #
# You should have received a copy of the GNU Affero Public License along      #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
#                                                                             #
###############################################################################


"""Usage:
  %(prog)s databases list [--remote]
31
  %(prog)s databases path [<name>]...
32
  %(prog)s databases edit <name>...
André Anjos's avatar
André Anjos committed
33
34
35
36
37
38
  %(prog)s databases check [<name>]...
  %(prog)s databases pull [--force] [<name>]...
  %(prog)s databases push [--force] [--dry-run] [<name>]...
  %(prog)s databases diff <name>
  %(prog)s databases status
  %(prog)s databases version <name>
39
40
  %(prog)s databases index [--list | --delete | --checksum] [--uid=<uid>] [--db-root=<path>] [--docker] [<name>]...
  %(prog)s databases view [--exclude=<output>] [--uid=<uid>] [--db-root=<path>] [--docker] <set_name>
André Anjos's avatar
André Anjos committed
41
42
43
44
45
46
47
48
49
50
  %(prog)s databases --help


Arguments:
  <name>           Database name formated as "<database>/<version>"
  <set_name>       Set formatted as "<database>/<version>/<protocol>/<set>"


Commands:
  list      Lists all the databases available on the platform
51
  path      Displays local path of databases files
52
  edit      Edit local database file
André Anjos's avatar
André Anjos committed
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
  check     Checks a local database for validity
  pull      Downloads the specified databases from the server
  push      Uploads databases to the server (must provide a valid admin token)
  diff      Shows changes between the local database and the remote version
  status    Shows (editing) status for all available databases
  version   Creates a new version of an existing database
  index     Indexes all outputs (of all sets) of a database.
  view      View the data of the specified dataset.


Options:
  --help              Display this screen
  --remote            Only acts on the remote copy of the database
  --exclude=<output>  When viewing, excludes this output
  --list              List index files matching output if they exist
  --delete            Delete index files matching output if they exist (also,
                      recursively deletes empty directories)
  --checksum          Checksums index files


Examples:

  To list all existing databases on your local prefix:

    $ %(prog)s db list

  To view the contents of a specific set

    $ %(prog)s db view simple/1/protocol/set

  To index the contents of a database

    $ %(prog)s db index simple/1

  To index the contents of a protocol on a database

    $ %(prog)s db index simple/1/double

  To index the contents of a set in a protocol on a database

    $ %(prog)s db index simple/1/double/double
"""

import os
import glob
98
import random
Samuel GAIST's avatar
Samuel GAIST committed
99
import zmq
100

André Anjos's avatar
André Anjos committed
101
import logging
Samuel GAIST's avatar
Samuel GAIST committed
102

André Anjos's avatar
André Anjos committed
103
104
105

import simplejson

Samuel GAIST's avatar
Samuel GAIST committed
106
107
from beat.core.hash import toPath
from beat.core.hash import hashDataset
André Anjos's avatar
André Anjos committed
108
109
from beat.core.utils import NumpyJSONEncoder
from beat.core.database import Database
110
from beat.core.data import load_data_index, RemoteDataSource
111
112
113
from beat.core import dock
from beat.core import inputs
from beat.core import utils
André Anjos's avatar
André Anjos committed
114
115
116

from . import common

Samuel GAIST's avatar
Samuel GAIST committed
117
logger = logging.getLogger(__name__)
André Anjos's avatar
André Anjos committed
118

119
120
121
122
CMD_DB_INDEX = 'index'
CMD_VIEW_OUTPUTS = 'databases_provider'


Samuel GAIST's avatar
Samuel GAIST committed
123
# ----------------------------------------------------------
124
125
126
127
128
129
130


def load_database_sets(configuration, database_name):
    # Process the name of the database
    parts = database_name.split('/')

    if len(parts) == 2:
Samuel GAIST's avatar
Samuel GAIST committed
131
132
133
        db_name = os.path.join(*parts[:2])
        protocol_filter = None
        set_filter = None
134
135

    elif len(parts) == 3:
Samuel GAIST's avatar
Samuel GAIST committed
136
137
138
        db_name = os.path.join(*parts[:2])
        protocol_filter = parts[2]
        set_filter = None
139
140

    elif len(parts) == 4:
Samuel GAIST's avatar
Samuel GAIST committed
141
142
143
        db_name = os.path.join(*parts[:2])
        protocol_filter = parts[2]
        set_filter = parts[3]
144
145

    else:
Samuel GAIST's avatar
Samuel GAIST committed
146
        logger.error("Database specification should have the format "
Samuel GAIST's avatar
Samuel GAIST committed
147
                     "`<database>/<version>/[<protocol>/[<set>]]', the value "
Samuel GAIST's avatar
Samuel GAIST committed
148
                     "you passed (%s) is not valid", database_name)
Samuel GAIST's avatar
Samuel GAIST committed
149
        return (None, None)
150
151
152

    # Load the dataformat
    dataformat_cache = {}
153
    database = Database(configuration.path,
Samuel GAIST's avatar
Samuel GAIST committed
154
                        db_name, dataformat_cache)
155
    if not database.valid:
Samuel GAIST's avatar
Samuel GAIST committed
156
157
158
159
        logger.error("Failed to load the database `%s':", db_name)
        for e in database.errors:
            logger.error('  * %s', e)
        return (None, None, None)
160
161
162
163
164

    # Filter the protocols
    protocols = database.protocol_names

    if protocol_filter is not None:
Samuel GAIST's avatar
Samuel GAIST committed
165
166
167
168
        if protocol_filter not in protocols:
            logger.error("The database `%s' does not have the protocol `%s' - "
                         "choose one of `%s'", db_name, protocol_filter,
                         ', '.join(protocols))
169

Samuel GAIST's avatar
Samuel GAIST committed
170
            return (None, None, None)
171

Samuel GAIST's avatar
Samuel GAIST committed
172
        protocols = [protocol_filter]
173
174
175
176
177

    # Filter the sets
    loaded_sets = []

    for protocol_name in protocols:
Samuel GAIST's avatar
Samuel GAIST committed
178
        sets = database.set_names(protocol_name)
179

Samuel GAIST's avatar
Samuel GAIST committed
180
181
182
        if set_filter is not None:
            if set_filter not in sets:
                logger.error("The database/protocol `%s/%s' does not have the "
Samuel GAIST's avatar
Samuel GAIST committed
183
184
185
                             "set `%s' - choose one of `%s'",
                             db_name, protocol_name, set_filter,
                             ', '.join(sets))
Samuel GAIST's avatar
Samuel GAIST committed
186
                return (None, None, None)
187

Samuel GAIST's avatar
Samuel GAIST committed
188
            sets = [z for z in sets if z == set_filter]
189

Samuel GAIST's avatar
Samuel GAIST committed
190
191
        loaded_sets.extend([(protocol_name, set_name,
                             database.set(protocol_name, set_name))
Samuel GAIST's avatar
Samuel GAIST committed
192
                            for set_name in sets])
193
194
195
196

    return (db_name, database, loaded_sets)


Samuel GAIST's avatar
Samuel GAIST committed
197
# ----------------------------------------------------------
198
199


200
201
def start_db_container(configuration, cmd, host,
                       db_name, protocol_name, set_name, database, db_set,
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
                       excluded_outputs=None, uid=None, db_root=None):

    input_list = inputs.InputList()

    input_group = inputs.InputGroup(set_name, restricted_access=False)
    input_list.add(input_group)

    db_configuration = {
        'inputs': {},
        'channel': set_name,
    }

    if uid is not None:
        db_configuration['datasets_uid'] = uid

    if db_root is not None:
        db_configuration['datasets_root_path'] = db_root

    for output_name, dataformat_name in db_set['outputs'].items():
Samuel GAIST's avatar
Samuel GAIST committed
221
        if excluded_outputs is not None and output_name in excluded_outputs:
222
223
            continue

224
        dataset_hash = hashDataset(db_name, protocol_name, set_name)
225
        db_configuration['inputs'][output_name] = dict(
Samuel GAIST's avatar
Samuel GAIST committed
226
227
228
229
230
231
232
            database=db_name,
            protocol=protocol_name,
            set=set_name,
            output=output_name,
            channel=set_name,
            hash=dataset_hash,
            path=toPath(dataset_hash, '.db')
233
234
235
236
237
238
239
240
241
242
243
        )

    db_tempdir = utils.temporary_directory()

    with open(os.path.join(db_tempdir, 'configuration.json'), 'wb') as f:
        simplejson.dump(db_configuration, f, indent=4)

    tmp_prefix = os.path.join(db_tempdir, 'prefix')
    if not os.path.exists(tmp_prefix):
        os.makedirs(tmp_prefix)

244
    database.export(tmp_prefix)
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260

    if db_root is None:
        json_path = os.path.join(tmp_prefix, 'databases', db_name + '.json')

        with open(json_path, 'r') as f:
            db_data = simplejson.load(f)

        database_path = db_data['root_folder']
        db_data['root_folder'] = os.path.join('/databases', db_name)

        with open(json_path, 'w') as f:
            simplejson.dump(db_data, f, indent=4)

    try:
        db_envkey = host.db2docker([db_name])
    except:
Samuel GAIST's avatar
Samuel GAIST committed
261
        raise RuntimeError("No environment found for the database `%s' "
262
263
264
265
266
267
                           "- available environments are %s" % (
                               db_name,
                               ", ".join(host.db_environments.keys())))

    # Creation of the container
    # Note: we only support one databases image loaded at the same time
268
269
270
271
272
    CONTAINER_PREFIX = '/beat/prefix'
    CONTAINER_CACHE = '/beat/cache'

    database_port = random.randint(51000, 60000)
    if cmd == CMD_VIEW_OUTPUTS:
Samuel GAIST's avatar
Samuel GAIST committed
273
274
275
276
277
278
        db_cmd = [
            cmd,
            '0.0.0.0:{}'.format(database_port),
            CONTAINER_PREFIX,
            CONTAINER_CACHE
        ]
279
    else:
Samuel GAIST's avatar
Samuel GAIST committed
280
281
282
283
284
285
286
287
        db_cmd = [
            cmd,
            CONTAINER_PREFIX,
            CONTAINER_CACHE,
            db_name,
            protocol_name,
            set_name
        ]
288
289

    databases_container = host.create_container(db_envkey, db_cmd)
290
    if cmd == CMD_VIEW_OUTPUTS:
Samuel GAIST's avatar
Samuel GAIST committed
291
292
        databases_container.add_port(
            database_port, database_port, host_address=host.ip)
293
294
    databases_container.add_volume(db_tempdir, '/beat/prefix')
    databases_container.add_volume(configuration.cache, '/beat/cache')
295
296

    # Specify the volumes to mount inside the container
Samuel GAIST's avatar
Samuel GAIST committed
297
    if 'datasets_root_path' not in db_configuration:
Samuel GAIST's avatar
Samuel GAIST committed
298
299
        databases_container.add_volume(
            database_path, os.path.join('/databases', db_name))
300
301
302
303
304
305
306
    else:
        databases_container.add_volume(db_configuration['datasets_root_path'],
                                       db_configuration['datasets_root_path'])

    # Start the container
    host.start(databases_container)

307
    if cmd == CMD_VIEW_OUTPUTS:
Samuel GAIST's avatar
Samuel GAIST committed
308
309
310
311
312
        # Communicate with container
        zmq_context = zmq.Context()
        db_socket = zmq_context.socket(zmq.PAIR)
        db_address = 'tcp://{}:{}'.format(host.ip, database_port)
        db_socket.connect(db_address)
313

Samuel GAIST's avatar
Samuel GAIST committed
314
        for output_name, dataformat_name in db_set['outputs'].items():
Samuel GAIST's avatar
Samuel GAIST committed
315
316
            if excluded_outputs is not None and \
               output_name in excluded_outputs:
Samuel GAIST's avatar
Samuel GAIST committed
317
                continue
318

Samuel GAIST's avatar
Samuel GAIST committed
319
320
321
            data_source = RemoteDataSource()
            data_source.setup(db_socket, output_name,
                              dataformat_name, configuration.path)
322

Samuel GAIST's avatar
Samuel GAIST committed
323
324
325
326
            input_ = inputs.Input(output_name,
                                  database.dataformats[dataformat_name],
                                  data_source)
            input_group.add(input_)
327

Samuel GAIST's avatar
Samuel GAIST committed
328
        return (databases_container, db_socket, zmq_context, input_list)
329
330

    return databases_container
331
332


Samuel GAIST's avatar
Samuel GAIST committed
333
# ----------------------------------------------------------
334
335


André Anjos's avatar
André Anjos committed
336
def pull(webapi, prefix, names, force, indentation, format_cache):
Samuel GAIST's avatar
Samuel GAIST committed
337
    """Copies databases (and required dataformats) from the server.
André Anjos's avatar
André Anjos committed
338

Samuel GAIST's avatar
Samuel GAIST committed
339
    Parameters:
André Anjos's avatar
André Anjos committed
340

Samuel GAIST's avatar
Samuel GAIST committed
341
342
      webapi (object): An instance of our WebAPI class, prepared to access the
        BEAT server of interest
André Anjos's avatar
André Anjos committed
343

Samuel GAIST's avatar
Samuel GAIST committed
344
345
      prefix (str): A string representing the root of the path in which the
        user objects are stored
André Anjos's avatar
André Anjos committed
346

André Anjos's avatar
André Anjos committed
347
348
349
350
351
      names (:py:class:`list`): A list of strings, each representing the unique
        relative path of the objects to retrieve or a list of usernames from
        which to retrieve objects. If the list is empty, then we pull all
        available objects of a given type. If no user is set, then pull all
        public objects of a given type.
André Anjos's avatar
André Anjos committed
352

Samuel GAIST's avatar
Samuel GAIST committed
353
354
      force (bool): If set to ``True``, then overwrites local changes with the
        remotely retrieved copies.
André Anjos's avatar
André Anjos committed
355

Samuel GAIST's avatar
Samuel GAIST committed
356
357
358
      indentation (int): The indentation level, useful if this function is
        called recursively while downloading different object types. This is
        normally set to ``0`` (zero).
André Anjos's avatar
André Anjos committed
359

Samuel GAIST's avatar
Samuel GAIST committed
360
361
      format_cache (dict): A dictionary containing all dataformats already
        downloaded.
André Anjos's avatar
André Anjos committed
362
363


Samuel GAIST's avatar
Samuel GAIST committed
364
    Returns:
André Anjos's avatar
André Anjos committed
365

Samuel GAIST's avatar
Samuel GAIST committed
366
367
      int: Indicating the exit status of the command, to be reported back to
        the calling process. This value should be zero if everything works OK,
Samuel GAIST's avatar
Samuel GAIST committed
368
        otherwise, different than zero (POSIX compliance).
André Anjos's avatar
André Anjos committed
369

Samuel GAIST's avatar
Samuel GAIST committed
370
    """
André Anjos's avatar
André Anjos committed
371

Samuel GAIST's avatar
Samuel GAIST committed
372
    from .dataformats import pull as dataformats_pull
André Anjos's avatar
André Anjos committed
373

Samuel GAIST's avatar
Samuel GAIST committed
374
    status, names = common.pull(webapi, prefix, 'database', names,
Samuel GAIST's avatar
Samuel GAIST committed
375
376
                                ['declaration', 'code', 'description'],
                                force, indentation)
André Anjos's avatar
André Anjos committed
377

Samuel GAIST's avatar
Samuel GAIST committed
378
379
380
381
382
    # see what dataformats one needs to pull
    dataformats = []
    for name in names:
        obj = Database(prefix, name)
        dataformats.extend(obj.dataformats.keys())
André Anjos's avatar
André Anjos committed
383

Samuel GAIST's avatar
Samuel GAIST committed
384
385
386
    # downloads any formats to which we depend on
    df_status = dataformats_pull(webapi, prefix, dataformats, force,
                                 indentation + 2, format_cache)
André Anjos's avatar
André Anjos committed
387

Samuel GAIST's avatar
Samuel GAIST committed
388
    return status + df_status
André Anjos's avatar
André Anjos committed
389
390


Samuel GAIST's avatar
Samuel GAIST committed
391
# ----------------------------------------------------------
André Anjos's avatar
André Anjos committed
392
393


394
def index_outputs(configuration, names, uid=None, db_root=None, docker=False):
André Anjos's avatar
André Anjos committed
395

396
397
    names = common.make_up_local_list(configuration.path, 'database', names)
    retcode = 0
André Anjos's avatar
André Anjos committed
398

Philip ABBET's avatar
Philip ABBET committed
399
    if docker:
Samuel GAIST's avatar
Samuel GAIST committed
400
        host = dock.Host(raise_on_errors=False)
André Anjos's avatar
André Anjos committed
401

402
    for database_name in names:
Samuel GAIST's avatar
Samuel GAIST committed
403
        logger.info("Indexing database %s...", database_name)
André Anjos's avatar
André Anjos committed
404

Samuel GAIST's avatar
Samuel GAIST committed
405
406
407
408
409
        (db_name, database, sets) = load_database_sets(
            configuration, database_name)
        if database is None:
            retcode += 1
            continue
André Anjos's avatar
André Anjos committed
410

Samuel GAIST's avatar
Samuel GAIST committed
411
412
        for protocol_name, set_name, db_set in sets:
            if not docker:
413
414
415
416
417
418
419
                try:
                    view = database.view(protocol_name, set_name)
                except SyntaxError as error:
                    logger.error("Failed to load the database `%s':",
                                 database_name)
                    logger.error('  * Syntax error: %s', error)
                    view = None
André Anjos's avatar
André Anjos committed
420

Samuel GAIST's avatar
Samuel GAIST committed
421
422
423
                if view is None:
                    retcode += 1
                    continue
424

Samuel GAIST's avatar
Samuel GAIST committed
425
                dataset_hash = hashDataset(db_name, protocol_name, set_name)
426
427
428
429
430
431
432
433
434
                try:
                    view.index(os.path.join(configuration.cache,
                                            toPath(dataset_hash, '.db')))
                except RuntimeError as error:
                  logger.error("Failed to load the database `%s':",
                               database_name)
                  logger.error('  * Runtime error %s', error)
                  retcode += 1
                  continue
435

Samuel GAIST's avatar
Samuel GAIST committed
436
437
438
            else:
                databases_container = \
                    start_db_container(configuration, CMD_DB_INDEX,
Samuel GAIST's avatar
Samuel GAIST committed
439
440
                                       host, db_name, protocol_name, set_name,
                                       database, db_set,
Samuel GAIST's avatar
Samuel GAIST committed
441
442
443
444
445
                                       uid=uid, db_root=db_root
                                       )
                status = host.wait(databases_container)
                if status != 0:
                    retcode += 1
André Anjos's avatar
André Anjos committed
446

447
    return retcode
André Anjos's avatar
André Anjos committed
448
449


Samuel GAIST's avatar
Samuel GAIST committed
450
# ----------------------------------------------------------
André Anjos's avatar
André Anjos committed
451
452


453
def list_index_files(configuration, names):
454

455
    names = common.make_up_local_list(configuration.path, 'database', names)
456

457
    retcode = 0
458

459
460
    for database_name in names:
        logger.info("Listing database %s indexes...", database_name)
461

Samuel GAIST's avatar
Samuel GAIST committed
462
463
        (db_name, database, sets) = load_database_sets(
            configuration, database_name)
464
465
466
        if database is None:
            retcode += 1
            continue
André Anjos's avatar
André Anjos committed
467

468
        for protocol_name, set_name, db_set in sets:
469
470
471
472
473
            dataset_hash = hashDataset(db_name, protocol_name, set_name)
            index_filename = toPath(dataset_hash)
            basename = os.path.splitext(index_filename)[0]
            for g in glob.glob(basename + '.*'):
                logger.info(g)
474

475
    return retcode
476
477


Samuel GAIST's avatar
Samuel GAIST committed
478
# ----------------------------------------------------------
479
480


481
def delete_index_files(configuration, names):
482

483
    names = common.make_up_local_list(configuration.path, 'database', names)
484

485
    retcode = 0
486

487
488
    for database_name in names:
        logger.info("Deleting database %s indexes...", database_name)
489

Samuel GAIST's avatar
Samuel GAIST committed
490
491
        (db_name, database, sets) = load_database_sets(
            configuration, database_name)
492
493
494
        if database is None:
            retcode += 1
            continue
495

496
497
        for protocol_name, set_name, db_set in sets:
            for output_name in db_set['outputs'].keys():
498
499
500
501
                dataset_hash = hashDataset(db_name, protocol_name, set_name)
                index_filename = toPath(dataset_hash)
                basename = os.path.join(configuration.cache,
                                        os.path.splitext(index_filename)[0])
502

503
504
505
                for g in glob.glob(basename + '.*'):
                    logger.info("removing `%s'...", g)
                    os.unlink(g)
506

507
508
                common.recursive_rmdir_if_empty(os.path.dirname(basename),
                                                configuration.cache)
509

510
    return retcode
511
512


Samuel GAIST's avatar
Samuel GAIST committed
513
# ----------------------------------------------------------
514

André Anjos's avatar
André Anjos committed
515

516
517
def view_outputs(configuration, dataset_name, excluded_outputs=None, uid=None,
                 db_root=None, docker=False):
André Anjos's avatar
André Anjos committed
518

519
520
    def data_to_json(data, indent):
        value = common.stringify(data.as_dict())
André Anjos's avatar
André Anjos committed
521

522
523
524
525
526
527
        value = simplejson.dumps(value, indent=4, cls=NumpyJSONEncoder) \
            .replace('"BEAT_LIST_DELIMITER[', '[') \
            .replace(']BEAT_LIST_DELIMITER"', ']') \
            .replace('"...",', '...') \
            .replace('"BEAT_LIST_SIZE(', '(') \
            .replace(')BEAT_LIST_SIZE"', ')')
André Anjos's avatar
André Anjos committed
528

529
        return ('\n' + ' ' * indent).join(value.split('\n'))
André Anjos's avatar
André Anjos committed
530

531
532
533
534
    # Load the infos about the database set
    (db_name, database, sets) = load_database_sets(configuration, dataset_name)
    if (database is None) or (len(sets) != 1):
        return 1
André Anjos's avatar
André Anjos committed
535

536
537
538
    (protocol_name, set_name, db_set) = sets[0]

    if excluded_outputs is not None:
Samuel GAIST's avatar
Samuel GAIST committed
539
540
        excluded_outputs = map(lambda x: x.strip(),
                               excluded_outputs.split(','))
André Anjos's avatar
André Anjos committed
541

542
543
    # Setup the view so the outputs can be used
    if not docker:
Samuel GAIST's avatar
Samuel GAIST committed
544
        view = database.view(protocol_name, set_name)
545

546
547
        if view is None:
            return 1
548
549
550
551
552
553
554

        dataset_hash = hashDataset(db_name, protocol_name, set_name)
        view.setup(os.path.join(configuration.cache,
                                toPath(dataset_hash, '.db')), pack=False)
        input_group = inputs.InputGroup(set_name, restricted_access=False)

        for output_name, dataformat_name in db_set['outputs'].items():
Samuel GAIST's avatar
Samuel GAIST committed
555
556
            if excluded_outputs is not None and \
               output_name in excluded_outputs:
557
558
                continue

Samuel GAIST's avatar
Samuel GAIST committed
559
560
561
            input = inputs.Input(output_name,
                                 database.dataformats[dataformat_name],
                                 view.data_sources[output_name])
562
563
            input_group.add(input)

564
565
566
567
    else:
        host = dock.Host(raise_on_errors=False)

        (databases_container, db_socket, zmq_context, input_list) = \
568
            start_db_container(configuration, CMD_VIEW_OUTPUTS,
Samuel GAIST's avatar
Samuel GAIST committed
569
570
571
572
                               host, db_name, protocol_name,
                               set_name, database, db_set,
                               excluded_outputs=excluded_outputs,
                               uid=uid, db_root=db_root)
573

574
        input_group = input_list.group(set_name)
André Anjos's avatar
André Anjos committed
575

576
577
578
    # Display the data
    try:
        previous_start = -1
André Anjos's avatar
André Anjos committed
579

580
581
        while input_group.hasMoreData():
            input_group.next()
André Anjos's avatar
André Anjos committed
582

583
584
            start = input_group.data_index
            end = input_group.data_index_end
André Anjos's avatar
André Anjos committed
585

586
587
            if start != previous_start:
                print(80 * '-')
André Anjos's avatar
André Anjos committed
588

589
                print('FROM %d TO %d' % (start, end))
André Anjos's avatar
André Anjos committed
590

Samuel GAIST's avatar
Samuel GAIST committed
591
592
593
                whole_inputs = [input_ for input_ in input_group
                                if input_.data_index == start and
                                input_.data_index_end == end]
André Anjos's avatar
André Anjos committed
594

595
596
                for input in whole_inputs:
                    label = ' - ' + str(input.name) + ': '
597
                    print(label + data_to_json(input.data, len(label)))
André Anjos's avatar
André Anjos committed
598

599
                previous_start = start
André Anjos's avatar
André Anjos committed
600

Samuel GAIST's avatar
Samuel GAIST committed
601
602
603
604
605
            selected_inputs = \
                [input_ for input_ in input_group
                 if input_.data_index == input_group.first_data_index and
                 (input_.data_index != start or
                  input_.data_index_end != end)]
André Anjos's avatar
André Anjos committed
606

607
            grouped_inputs = {}
Samuel GAIST's avatar
Samuel GAIST committed
608
609
610
            for input_ in selected_inputs:
                key = (input_.data_index, input_.data_index_end)
                if key not in grouped_inputs:
611
612
                    grouped_inputs[key] = []
                grouped_inputs[key].append(input)
André Anjos's avatar
André Anjos committed
613

614
            sorted_keys = sorted(grouped_inputs.keys())
615
616
617

            for key in sorted_keys:
                print
618
                print('  FROM %d TO %d' % key)
619
620
621

                for input in grouped_inputs[key]:
                    label = '   - ' + str(input.name) + ': '
622
                    print(label + data_to_json(input.data, len(label)))
André Anjos's avatar
André Anjos committed
623
624

    except Exception as e:
625
626
627
628
        logger.error("Failed to retrieve the next data: %s", e)
        return 1

    return 0
André Anjos's avatar
André Anjos committed
629

630

Samuel GAIST's avatar
Samuel GAIST committed
631
# ----------------------------------------------------------
André Anjos's avatar
André Anjos committed
632
633
634
635


def process(args):

636
637
638
639
    uid = int(args['--uid']) if args['--uid'] is not None else None
    configuration = args['config']
    db_names = args['<name>']

Samuel GAIST's avatar
Samuel GAIST committed
640
641
    if args['list']:
        if args['--remote']:
642
            with common.make_webapi(configuration) as webapi:
Samuel GAIST's avatar
Samuel GAIST committed
643
644
                return common.display_remote_list(webapi, 'database')
        else:
645
            return common.display_local_list(configuration.path, 'database')
Samuel GAIST's avatar
Samuel GAIST committed
646

647
    elif args['path']:
648
649
        return common.display_local_path(configuration.path, 'database', db_names)

650
    elif args['edit']:
651
      return common.edit_local_file(configuration.path, configuration.editor, 'database', db_names[0])
652

Samuel GAIST's avatar
Samuel GAIST committed
653
    elif args['check']:
654
        return common.check(configuration.path, 'database', db_names)
Samuel GAIST's avatar
Samuel GAIST committed
655
656

    elif args['pull']:
657
658
        with common.make_webapi(configuration) as webapi:
            return pull(webapi, configuration.path, db_names,
Samuel GAIST's avatar
Samuel GAIST committed
659
660
661
                        args['--force'], 0, {})

    elif args['push']:
662
663
664
665
        with common.make_webapi(configuration) as webapi:
            return common.push(webapi, configuration.path, 'database',
                               db_names, ['name', 'declaration',
                                          'code', 'description'],
Samuel GAIST's avatar
Samuel GAIST committed
666
667
668
                               {}, args['--force'], args['--dry-run'], 0)

    elif args['diff']:
669
670
671
672
        with common.make_webapi(configuration) as webapi:
            return common.diff(webapi, configuration.path, 'database',
                               db_names[0],
                               ['declaration', 'code', 'description'])
Samuel GAIST's avatar
Samuel GAIST committed
673
674

    elif args['status']:
675
676
        with common.make_webapi(configuration) as webapi:
            return common.status(webapi, configuration.path, 'database')[0]
Samuel GAIST's avatar
Samuel GAIST committed
677
678

    elif args['version']:
679
680
        return common.new_version(configuration.path, 'database',
                                  db_names[0])
Samuel GAIST's avatar
Samuel GAIST committed
681
682
683

    elif args['view']:
        if args['--exclude']:
684
685
686
687
688
            return view_outputs(configuration, args['<set_name>'],
                                args['--exclude'],
                                uid=uid,
                                db_root=args['--db-root'],
                                docker=args['--docker'])
Samuel GAIST's avatar
Samuel GAIST committed
689
        else:
690
691
692
693
            return view_outputs(configuration, args['<set_name>'],
                                uid=uid,
                                db_root=args['--db-root'],
                                docker=args['--docker'])
Samuel GAIST's avatar
Samuel GAIST committed
694
695
696

    elif args['index']:
        if args['--list']:
697
            return list_index_files(configuration, db_names)
Samuel GAIST's avatar
Samuel GAIST committed
698
        elif args['--delete']:
699
            return delete_index_files(configuration, db_names)
Samuel GAIST's avatar
Samuel GAIST committed
700
        else:
701
702
703
704
            return index_outputs(configuration, db_names,
                                 uid=uid,
                                 db_root=args['--db-root'],
                                 docker=args['--docker'])
Samuel GAIST's avatar
Samuel GAIST committed
705
706
707
708

    # Should not happen
    logger.error("unrecognized `databases' subcommand")
    return 1