database.py 22.9 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

###############################################################################
#                                                                             #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
# Contact: beat.support@idiap.ch                                              #
#                                                                             #
# This file is part of the beat.backend.python module of the BEAT platform.   #
#                                                                             #
# Commercial License Usage                                                    #
# Licensees holding valid commercial BEAT licenses may use this file in       #
# accordance with the terms contained in a written agreement between you      #
# and Idiap. For further information contact tto@idiap.ch                     #
#                                                                             #
# Alternatively, this file may be used under the terms of the GNU Affero      #
# Public License version 3 as published by the Free Software and appearing    #
# in the file LICENSE.AGPL included in the packaging of this file.            #
# The BEAT platform is distributed in the hope that it will be useful, but    #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
#                                                                             #
# You should have received a copy of the GNU Affero Public License along      #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
#                                                                             #
###############################################################################


"""Validation of databases"""

import os
import sys

import six
import simplejson
36 37
import itertools
import numpy as np
38 39

from . import loader
Philip ABBET's avatar
Philip ABBET committed
40 41 42
from . import utils

from .dataformat import DataFormat
43
from .outputs import OutputList
Philip ABBET's avatar
Philip ABBET committed
44 45


46 47
#----------------------------------------------------------

Philip ABBET's avatar
Philip ABBET committed
48 49

class Storage(utils.CodeStorage):
Philip ABBET's avatar
Philip ABBET committed
50
    """Resolves paths for databases
Philip ABBET's avatar
Philip ABBET committed
51

Philip ABBET's avatar
Philip ABBET committed
52
    Parameters:
Philip ABBET's avatar
Philip ABBET committed
53

Philip ABBET's avatar
Philip ABBET committed
54
      prefix (str): Establishes the prefix of your installation.
Philip ABBET's avatar
Philip ABBET committed
55

Philip ABBET's avatar
Philip ABBET committed
56 57
      name (str): The name of the database object in the format
        ``<name>/<version>``.
Philip ABBET's avatar
Philip ABBET committed
58

Philip ABBET's avatar
Philip ABBET committed
59
    """
Philip ABBET's avatar
Philip ABBET committed
60

Philip ABBET's avatar
Philip ABBET committed
61
    def __init__(self, prefix, name):
Philip ABBET's avatar
Philip ABBET committed
62

Philip ABBET's avatar
Philip ABBET committed
63 64
        if name.count('/') != 1:
            raise RuntimeError("invalid database name: `%s'" % name)
Philip ABBET's avatar
Philip ABBET committed
65

Philip ABBET's avatar
Philip ABBET committed
66 67
        self.name, self.version = name.split('/')
        self.fullname = name
Philip ABBET's avatar
Philip ABBET committed
68

Philip ABBET's avatar
Philip ABBET committed
69 70
        path = os.path.join(prefix, 'databases', name)
        super(Storage, self).__init__(path, 'python') #views are coded in Python
71 72


73 74
#----------------------------------------------------------

75 76

class View(object):
Philip ABBET's avatar
Philip ABBET committed
77
    '''A special loader class for database views, with specialized methods
78

Philip ABBET's avatar
Philip ABBET committed
79
    Parameters:
80

Philip ABBET's avatar
Philip ABBET committed
81
      db_name (str): The full name of the database object for this view
82

Philip ABBET's avatar
Philip ABBET committed
83 84
      module (module): The preloaded module containing the database views as
        returned by :py:func:`beat.core.loader.load_module`.
85

Philip ABBET's avatar
Philip ABBET committed
86
      prefix (str, path): The prefix path for the current installation
87

Philip ABBET's avatar
Philip ABBET committed
88 89
      root_folder (str, path): The path pointing to the root folder of this
        database
90

Philip ABBET's avatar
Philip ABBET committed
91 92 93
      exc (class): The class to use as base exception when translating the
        exception from the user code. Read the documention of :py:func:`run`
        for more details.
94

Philip ABBET's avatar
Philip ABBET committed
95
      *args: Constructor parameters for the database view. Normally, none.
96

Philip ABBET's avatar
Philip ABBET committed
97
      **kwargs: Constructor parameters for the database view. Normally, none.
98

Philip ABBET's avatar
Philip ABBET committed
99
    '''
100 101


Philip ABBET's avatar
Philip ABBET committed
102 103
    def __init__(self, module, definition, prefix, root_folder, exc=None,
            *args, **kwargs):
104

Philip ABBET's avatar
Philip ABBET committed
105 106 107 108 109 110 111 112
        try:
            class_ = getattr(module, definition['view'])
        except Exception as e:
            if exc is not None:
                type, value, traceback = sys.exc_info()
                six.reraise(exc, exc(value), traceback)
            else:
                raise #just re-raise the user exception
113

Philip ABBET's avatar
Philip ABBET committed
114 115 116 117 118 119 120
        self.obj = loader.run(class_, '__new__', exc, *args, **kwargs)
        self.ready = False
        self.prefix = prefix
        self.root_folder = root_folder
        self.definition = definition
        self.exc = exc or RuntimeError
        self.outputs = None
121 122


Philip ABBET's avatar
Philip ABBET committed
123 124
    def prepare_outputs(self):
        '''Prepares the outputs of the dataset'''
125

Philip ABBET's avatar
Philip ABBET committed
126 127 128
        from .outputs import Output, OutputList
        from .data import MemoryDataSink
        from .dataformat import DataFormat
129

Philip ABBET's avatar
Philip ABBET committed
130 131 132 133 134 135 136 137
        # create the stock outputs for this dataset, so data is dumped
        # on a in-memory sink
        self.outputs = OutputList()
        for out_name, out_format in self.definition.get('outputs', {}).items():
            data_sink = MemoryDataSink()
            data_sink.dataformat = DataFormat(self.prefix, out_format)
            data_sink.setup([])
            self.outputs.add(Output(out_name, data_sink, dataset_output=True))
138 139


Philip ABBET's avatar
Philip ABBET committed
140 141
    def setup(self, *args, **kwargs):
        '''Sets up the view'''
142

Philip ABBET's avatar
Philip ABBET committed
143 144
        kwargs.setdefault('root_folder', self.root_folder)
        kwargs.setdefault('parameters', self.definition.get('parameters', {}))
145

Philip ABBET's avatar
Philip ABBET committed
146 147 148 149
        if 'outputs' not in kwargs:
            kwargs['outputs'] = self.outputs
        else:
            self.outputs = kwargs['outputs'] #record outputs nevertheless
150

Philip ABBET's avatar
Philip ABBET committed
151
        self.ready = loader.run(self.obj, 'setup', self.exc, *args, **kwargs)
152

Philip ABBET's avatar
Philip ABBET committed
153 154
        if not self.ready:
            raise self.exc("unknow setup failure")
155

Philip ABBET's avatar
Philip ABBET committed
156
        return self.ready
157 158


Philip ABBET's avatar
Philip ABBET committed
159 160
    def input_group(self, name='default', exclude_outputs=[]):
        '''A memory-source input group matching the outputs from the view'''
161

Philip ABBET's avatar
Philip ABBET committed
162 163
        if not self.ready:
            raise self.exc("database view not yet setup")
164

Philip ABBET's avatar
Philip ABBET committed
165 166 167
        from .data import MemoryDataSource
        from .outputs import SynchronizationListener
        from .inputs import Input, InputGroup
168

Philip ABBET's avatar
Philip ABBET committed
169 170 171 172 173
        # Setup the inputs
        synchronization_listener = SynchronizationListener()
        input_group = InputGroup(name,
                synchronization_listener=synchronization_listener,
                restricted_access=False)
174

Philip ABBET's avatar
Philip ABBET committed
175 176 177 178 179 180
        for output in self.outputs:
            if output.name in exclude_outputs: continue
            data_source = MemoryDataSource(self.done, next_callback=self.next)
            output.data_sink.data_sources.append(data_source)
            input_group.add(Input(output.name,
                output.data_sink.dataformat, data_source))
181

Philip ABBET's avatar
Philip ABBET committed
182
        return input_group
183 184


Philip ABBET's avatar
Philip ABBET committed
185 186
    def done(self, *args, **kwargs):
        '''Checks if the view is done'''
187

Philip ABBET's avatar
Philip ABBET committed
188 189
        if not self.ready:
            raise self.exc("database view not yet setup")
190

Philip ABBET's avatar
Philip ABBET committed
191
        return loader.run(self.obj, 'done', self.exc, *args, **kwargs)
192 193


Philip ABBET's avatar
Philip ABBET committed
194 195
    def next(self, *args, **kwargs):
        '''Runs through the next data chunk'''
196

Philip ABBET's avatar
Philip ABBET committed
197 198 199
        if not self.ready:
            raise self.exc("database view not yet setup")
        return loader.run(self.obj, 'next', self.exc, *args, **kwargs)
200 201


Philip ABBET's avatar
Philip ABBET committed
202 203 204
    def __getattr__(self, key):
        '''Returns an attribute of the view - only called at last resort'''
        return getattr(self.obj, key)
205 206


207 208
#----------------------------------------------------------

209 210

class Database(object):
Philip ABBET's avatar
Philip ABBET committed
211
    """Databases define the start point of the dataflow in an experiment.
212 213


Philip ABBET's avatar
Philip ABBET committed
214
    Parameters:
215

Philip ABBET's avatar
Philip ABBET committed
216
      prefix (str): Establishes the prefix of your installation.
217

Philip ABBET's avatar
Philip ABBET committed
218
      name (str): The fully qualified database name (e.g. ``db/1``)
219

Philip ABBET's avatar
Philip ABBET committed
220 221 222 223 224 225
      dataformat_cache (dict, optional): A dictionary mapping dataformat names
        to loaded dataformats. This parameter is optional and, if passed, may
        greatly speed-up database loading times as dataformats that are already
        loaded may be re-used. If you use this parameter, you must guarantee
        that the cache is refreshed as appropriate in case the underlying
        dataformats change.
226 227


Philip ABBET's avatar
Philip ABBET committed
228
    Attributes:
229

Philip ABBET's avatar
Philip ABBET committed
230
      name (str): The full, valid name of this database
231

Philip ABBET's avatar
Philip ABBET committed
232 233
      data (dict): The original data for this database, as loaded by our JSON
        decoder.
234

Philip ABBET's avatar
Philip ABBET committed
235
    """
236

Philip ABBET's avatar
Philip ABBET committed
237
    def __init__(self, prefix, name, dataformat_cache=None):
238

Philip ABBET's avatar
Philip ABBET committed
239 240 241 242
        self._name = None
        self.prefix = prefix
        self.dataformats = {} # preloaded dataformats
        self.storage = None
243

Philip ABBET's avatar
Philip ABBET committed
244 245
        self.errors = []
        self.data = None
246

Philip ABBET's avatar
Philip ABBET committed
247 248
        # if the user has not provided a cache, still use one for performance
        dataformat_cache = dataformat_cache if dataformat_cache is not None else {}
249

Philip ABBET's avatar
Philip ABBET committed
250
        self._load(name, dataformat_cache)
251 252


Philip ABBET's avatar
Philip ABBET committed
253 254
    def _load(self, data, dataformat_cache):
        """Loads the database"""
255

Philip ABBET's avatar
Philip ABBET committed
256
        self._name = data
Philip ABBET's avatar
Philip ABBET committed
257

Philip ABBET's avatar
Philip ABBET committed
258 259 260 261 262
        self.storage = Storage(self.prefix, self._name)
        json_path = self.storage.json.path
        if not self.storage.json.exists():
            self.errors.append('Database declaration file not found: %s' % json_path)
            return
Philip ABBET's avatar
Philip ABBET committed
263

Philip ABBET's avatar
Philip ABBET committed
264 265
        with open(json_path, 'rb') as f:
            self.data = simplejson.load(f)
Philip ABBET's avatar
Philip ABBET committed
266

Philip ABBET's avatar
Philip ABBET committed
267 268
        for protocol in self.data['protocols']:
            for _set in protocol['sets']:
Philip ABBET's avatar
Philip ABBET committed
269

Philip ABBET's avatar
Philip ABBET committed
270
                for key, value in _set['outputs'].items():
Philip ABBET's avatar
Philip ABBET committed
271

Philip ABBET's avatar
Philip ABBET committed
272 273
                    if value in self.dataformats:
                        continue
Philip ABBET's avatar
Philip ABBET committed
274

Philip ABBET's avatar
Philip ABBET committed
275 276 277 278 279
                    if value in dataformat_cache:
                        dataformat = dataformat_cache[value]
                    else:
                        dataformat = DataFormat(self.prefix, value)
                        dataformat_cache[value] = dataformat
Philip ABBET's avatar
Philip ABBET committed
280

Philip ABBET's avatar
Philip ABBET committed
281
                    self.dataformats[value] = dataformat
282 283


Philip ABBET's avatar
Philip ABBET committed
284 285 286 287 288
    @property
    def name(self):
        """Returns the name of this object
        """
        return self._name or '__unnamed_database__'
289 290


Philip ABBET's avatar
Philip ABBET committed
291 292 293 294
    @property
    def schema_version(self):
        """Returns the schema version"""
        return self.data.get('schema_version', 1)
295 296


Philip ABBET's avatar
Philip ABBET committed
297 298 299
    @property
    def valid(self):
        return not bool(self.errors)
Philip ABBET's avatar
Philip ABBET committed
300 301


Philip ABBET's avatar
Philip ABBET committed
302 303 304
    @property
    def protocols(self):
        """The declaration of all the protocols of the database"""
305

Philip ABBET's avatar
Philip ABBET committed
306 307
        data = self.data['protocols']
        return dict(zip([k['name'] for k in data], data))
308 309


Philip ABBET's avatar
Philip ABBET committed
310 311
    def protocol(self, name):
        """The declaration of a specific protocol in the database"""
312

Philip ABBET's avatar
Philip ABBET committed
313
        return self.protocols[name]
314 315


Philip ABBET's avatar
Philip ABBET committed
316 317 318
    @property
    def protocol_names(self):
        """Names of protocols declared for this database"""
319

Philip ABBET's avatar
Philip ABBET committed
320 321
        data = self.data['protocols']
        return [k['name'] for k in data]
322 323


Philip ABBET's avatar
Philip ABBET committed
324 325
    def sets(self, protocol):
        """The declaration of a specific set in the database protocol"""
326

Philip ABBET's avatar
Philip ABBET committed
327 328
        data = self.protocol(protocol)['sets']
        return dict(zip([k['name'] for k in data], data))
329 330


Philip ABBET's avatar
Philip ABBET committed
331 332
    def set(self, protocol, name):
        """The declaration of all the protocols of the database"""
333

Philip ABBET's avatar
Philip ABBET committed
334
        return self.sets(protocol)[name]
335 336


Philip ABBET's avatar
Philip ABBET committed
337 338
    def set_names(self, protocol):
        """The names of sets in a given protocol for this database"""
339

Philip ABBET's avatar
Philip ABBET committed
340 341
        data = self.protocol(protocol)['sets']
        return [k['name'] for k in data]
342 343


Philip ABBET's avatar
Philip ABBET committed
344 345
    def view(self, protocol, name, exc=None):
        """Returns the database view, given the protocol and the set name
346

Philip ABBET's avatar
Philip ABBET committed
347
        Parameters:
348

Philip ABBET's avatar
Philip ABBET committed
349
          protocol (str): The name of the protocol where to retrieve the view from
350

Philip ABBET's avatar
Philip ABBET committed
351 352
          name (str): The name of the set in the protocol where to retrieve the
            view from
353

Philip ABBET's avatar
Philip ABBET committed
354 355
          exc (class): If passed, must be a valid exception class that will be
            used to report errors in the read-out of this database's view.
356

Philip ABBET's avatar
Philip ABBET committed
357
        Returns:
358

Philip ABBET's avatar
Philip ABBET committed
359 360
          The database view, which will be constructed, but not setup. You
          **must** set it up before using methods ``done`` or ``next``.
361

Philip ABBET's avatar
Philip ABBET committed
362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384
        """

        if not self._name:
            exc = exc or RuntimeError
            raise exc("database has no name")

        if not self.valid:
            message = "cannot load view for set `%s' of protocol `%s' " \
                    "from invalid database (%s)" % (protocol, name, self.name)
            if exc: raise exc(message)
            raise RuntimeError(message)

        # loads the module only once through the lifetime of the database object
        try:
            if not hasattr(self, '_module'):
                self._module = loader.load_module(self.name.replace(os.sep, '_'),
                          self.storage.code.path, {})
        except Exception as e:
            if exc is not None:
                type, value, traceback = sys.exc_info()
                six.reraise(exc, exc(value), traceback)
            else:
                raise #just re-raise the user exception
385

Philip ABBET's avatar
Philip ABBET committed
386 387
        return View(self._module, self.set(protocol, name), self.prefix,
                self.data['root_folder'], exc)
388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662


#----------------------------------------------------------


class DatabaseTester:
    """Used while developing a new database view, to test its behavior

    This class tests that, for each combination of connected/not connected
    outputs:

      - Data indices seems consistent
      - All the connected outputs produce data
      - All the not connected outputs don't produce data

    It also report some stats, and can generate a text file detailing the
    data generated by each output.

    By default, outputs are assumed to produce data at constant intervals.
    Those that don't follow this pattern, must be declared as 'irregular'.

    Note that no particular check is done about the database declaration or
    the correctness of the generated data with their data formats. This class
    is mainly used to check that the outputs are correctly synchronized.
    """

    # Mock output class
    class MockOutput:

        def __init__(self, name, connected):
            self.name = name
            self.connected = connected
            self.last_written_data_index = -1
            self.written_data = []

        def write(self, data, end_data_index):
            self.written_data.append(( self.last_written_data_index + 1, end_data_index, data ))
            self.last_written_data_index = end_data_index

        def isConnected(self):
            return self.connected


    class SynchronizedUnit:

        def __init__(self, start, end):
            self.start = start
            self.end = end
            self.data = {}
            self.children = []

        def addData(self, output, start, end, data):
            if (start == self.start) and (end == self.end):
                self.data[output] = self._dataToString(data)
            elif (len(self.children) == 0) or (self.children[-1].end < start):
                unit = DatabaseTester.SynchronizedUnit(start, end)
                unit.addData(output, start, end, data)
                self.children.append(unit)
            else:
                for index, unit in enumerate(self.children):
                    if (unit.start <= start) and (unit.end >= end):
                        unit.addData(output, start, end, data)
                        break
                    elif (unit.start == start) and (unit.end < end):
                        new_unit = DatabaseTester.SynchronizedUnit(start, end)
                        new_unit.addData(output, start, end, data)
                        new_unit.children.append(unit)

                        for i in range(index + 1, len(self.children)):
                            unit = self.children[i]
                            if (unit.end <= end):
                                new_unit.children.append(unit)
                            else:
                                break

                        self.children = self.children[:index] + [new_unit] + self.children[i:]
                        break

        def toString(self):
            texts = {}

            for child in self.children:
                child_texts = child.toString()
                for output, text in child_texts.items():
                    if texts.has_key(output):
                        texts[output] += ' ' + text
                    else:
                        texts[output] = text

            if len(self.data) > 0:
                length = max([ len(x) + 6 for x in self.data.values() ])

                if len(texts) > 0:
                    children_length = len(texts.values()[0])
                    if children_length >= length:
                        length = children_length
                    else:
                        diff = length - children_length
                        if diff % 2 == 0:
                            diff1 = diff / 2
                            diff2 = diff1
                        else:
                            diff1 = diff // 2
                            diff2 = diff - diff1

                        for k, v in texts.items():
                            texts[k] = '|%s%s%s|' % ('-' * diff1, v[1:-1], '-' * diff2)

                for output, value in self.data.items():
                    output_length = len(value) + 6
                    diff = length - output_length
                    if diff % 2 == 0:
                        diff1 = diff / 2
                        diff2 = diff1
                    else:
                        diff1 = diff // 2
                        diff2 = diff - diff1
                    texts[output] = '|-%s %s %s-|' % ('-' * diff1, value, '-' * diff2)

            length = max(len(x) for x in texts.values())
            for k, v in texts.items():
                if len(v) < length:
                    texts[k] += ' ' * (length - len(v))

            return texts

        def _dataToString(self, data):
            if (len(data) > 1) or (len(data) == 0):
                return 'X'

            value = data[data.keys()[0]]

            if isinstance(value, np.ndarray) or isinstance(value, dict):
                return 'X'

            return str(value)


    def __init__(self, name, view_class, outputs_declaration, parameters,
                 irregular_outputs=[], all_combinations=True):
        self.name = name
        self.view_class = view_class
        self.outputs_declaration = {}
        self.parameters = parameters
        self.irregular_outputs = irregular_outputs

        self.determine_regular_intervals(outputs_declaration)

        if all_combinations:
            for L in range(0, len(self.outputs_declaration) + 1):
                for subset in itertools.combinations(self.outputs_declaration.keys(), L):
                    self.run(subset)
        else:
            self.run(self.outputs_declaration.keys())

        print

    def determine_regular_intervals(self, outputs_declaration):
        outputs = OutputList()
        for name in outputs_declaration:
            outputs.add(DatabaseTester.MockOutput(name, True))

        view = self.view_class()
        view.setup('', outputs, self.parameters)

        view.next()

        for output in outputs:
            if output.name not in self.irregular_outputs:
                self.outputs_declaration[output.name] = output.last_written_data_index + 1
            else:
                self.outputs_declaration[output.name] = None


    def run(self, connected_outputs):
        if len(connected_outputs) == 0:
            return

        print "Testing '%s', with %d output(s): %s" % (self.name, len(connected_outputs),
                                                       ', '.join(connected_outputs))

        # Create the mock outputs
        connected_outputs = dict([ x for x in self.outputs_declaration.items()
                                     if x[0] in connected_outputs ])

        not_connected_outputs = dict([ x for x in self.outputs_declaration.items()
                                         if x[0] not in connected_outputs ])

        outputs = OutputList()
        for name in self.outputs_declaration.keys():
            outputs.add(DatabaseTester.MockOutput(name, name in connected_outputs))


        # Create the view
        view = self.view_class()
        view.setup('', outputs, self.parameters)


        # Initialisations
        next_expected_indices = {}
        for name, interval in connected_outputs.items():
            next_expected_indices[name] = 0

        next_index = 0

        def _done():
            for output in outputs:
                if output.isConnected() and not view.done(output.last_written_data_index):
                    return False
            return True


        # Ask for all the data
        while not(_done()):
            view.next()

            # Check the indices for the connected outputs
            for name in connected_outputs.keys():
                if name not in self.irregular_outputs:
                    assert(outputs[name].written_data[-1][0] == next_expected_indices[name])
                    assert(outputs[name].written_data[-1][1] == next_expected_indices[name] + connected_outputs[name] - 1)
                else:
                    assert(outputs[name].written_data[-1][0] == next_expected_indices[name])
                    assert(outputs[name].written_data[-1][1] >= next_expected_indices[name])

            # Check that the not connected outputs didn't produce data
            for name in not_connected_outputs.keys():
                assert(len(outputs[name].written_data) == 0)

            # Compute the next data index that should be produced
            next_index = 1 + min([ x.written_data[-1][1] for x in outputs if x.isConnected() ])

            # Compute the next data index that should be produced by each connected output
            for name in connected_outputs.keys():
                if name not in self.irregular_outputs:
                    if next_index == next_expected_indices[name] + connected_outputs[name]:
                        next_expected_indices[name] += connected_outputs[name]
                else:
                    if next_index > outputs[name].written_data[-1][1]:
                        next_expected_indices[name] = outputs[name].written_data[-1][1] + 1

        # Check the number of data produced on the regular outputs
        for name in connected_outputs.keys():
            print '  - %s: %d data' % (name, len(outputs[name].written_data))
            if name not in self.irregular_outputs:
                assert(len(outputs[name].written_data) == next_index / connected_outputs[name])

        # Check that all outputs ends on the same index
        for name in connected_outputs.keys():
            assert(outputs[name].written_data[-1][1] == next_index - 1)


        # Generate a text file with lots of details (only if all outputs are connected)
        if len(connected_outputs) == len(self.outputs_declaration):
            sorted_outputs = sorted(outputs, key=lambda x: len(x.written_data))

            unit = DatabaseTester.SynchronizedUnit(0, sorted_outputs[0].written_data[-1][1])

            for output in sorted_outputs:
                for data in output.written_data:
                    unit.addData(output.name, data[0], data[1], data[2])

            texts = unit.toString()

            outputs_max_length = max([ len(x) for x in self.outputs_declaration.keys() ])

            with open(self.name.replace(' ', '_') + '.txt', 'w') as f:
                for i in range(1, len(sorted_outputs) + 1):
                    output_name = sorted_outputs[-i].name
                    f.write(output_name + ': ')

                    if len(output_name) < outputs_max_length:
                        f.write(' ' * (outputs_max_length - len(output_name)))

                    f.write(texts[output_name] + '\n')