test_experiment_loading.py 7.28 KB
Newer Older
André Anjos's avatar
André Anjos committed
1
2
3
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

Samuel GAIST's avatar
Samuel GAIST committed
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
###################################################################################
#                                                                                 #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/               #
# Contact: beat.support@idiap.ch                                                  #
#                                                                                 #
# Redistribution and use in source and binary forms, with or without              #
# modification, are permitted provided that the following conditions are met:     #
#                                                                                 #
# 1. Redistributions of source code must retain the above copyright notice, this  #
# list of conditions and the following disclaimer.                                #
#                                                                                 #
# 2. Redistributions in binary form must reproduce the above copyright notice,    #
# this list of conditions and the following disclaimer in the documentation       #
# and/or other materials provided with the distribution.                          #
#                                                                                 #
# 3. Neither the name of the copyright holder nor the names of its contributors   #
# may be used to endorse or promote products derived from this software without   #
# specific prior written permission.                                              #
#                                                                                 #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED   #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE          #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE    #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL      #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR      #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER      #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,   #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE   #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.            #
#                                                                                 #
###################################################################################
André Anjos's avatar
André Anjos committed
35
36
37
38
39
40
41
42
43


import six
import nose.tools
from ..experiment import Experiment

from . import prefix, tmp_prefix
from .utils import cleanup

Samuel GAIST's avatar
Samuel GAIST committed
44

45
46
47
48
49
50
51
52
53
54
55
# ----------------------------------------------------------


@nose.tools.raises(RuntimeError)
def test_load_default_experiment_fails():
    Experiment(prefix, data=None)


# ----------------------------------------------------------


André Anjos's avatar
André Anjos committed
56
57
def test_load_valid_experiment():

Samuel GAIST's avatar
Samuel GAIST committed
58
    experiment = Experiment(prefix, "user/integers_addition/1/integers_addition")
André Anjos's avatar
André Anjos committed
59

60
61
62
    nose.tools.assert_true(
        experiment.valid, "\n  * %s" % "\n  * ".join(experiment.errors)
    )
Samuel GAIST's avatar
Samuel GAIST committed
63
    nose.tools.eq_(experiment.label, "user/user/integers_addition/1/integers_addition")
André Anjos's avatar
André Anjos committed
64

65
66
67
    nose.tools.assert_true(
        experiment.toolchain.valid,
        "\n  * %s" % "\n  * ".join(experiment.toolchain.errors),
Samuel GAIST's avatar
Samuel GAIST committed
68
69
    )
    nose.tools.eq_(experiment.toolchain.name, "user/integers_addition/1")
André Anjos's avatar
André Anjos committed
70

Philip ABBET's avatar
Philip ABBET committed
71
    nose.tools.eq_(len(experiment.datasets), 1)
72
    nose.tools.assert_true("integers" in experiment.datasets)
Philip ABBET's avatar
Philip ABBET committed
73
74
75
    nose.tools.eq_(len(experiment.databases), 1)
    nose.tools.eq_(len(experiment.blocks), 1)
    nose.tools.eq_(len(experiment.analyzers), 1)
André Anjos's avatar
André Anjos committed
76

Philip ABBET's avatar
Philip ABBET committed
77
    nose.tools.eq_(len(experiment.algorithms), 2)
78
79
80
81
    nose.tools.assert_true("user/sum/1" in experiment.algorithms)
    nose.tools.assert_true(experiment.algorithms["user/sum/1"].valid)
    nose.tools.assert_true("user/integers_analysis/1" in experiment.algorithms)
    nose.tools.assert_true(experiment.algorithms["user/integers_analysis/1"].valid)
Samuel GAIST's avatar
Samuel GAIST committed
82

André Anjos's avatar
André Anjos committed
83
84
85

def test_load_one_dataset_two_blocks_toolchain():

Samuel GAIST's avatar
Samuel GAIST committed
86
    experiment = Experiment(prefix, "user/integers_addition/2/integers_addition")
87
88
89
    nose.tools.assert_true(
        experiment.valid, "\n  * %s" % "\n  * ".join(experiment.errors)
    )
Samuel GAIST's avatar
Samuel GAIST committed
90
    nose.tools.eq_(experiment.label, "user/user/integers_addition/2/integers_addition")
André Anjos's avatar
André Anjos committed
91

92
93
94
    nose.tools.assert_true(
        experiment.toolchain.valid,
        "\n  * %s" % "\n  * ".join(experiment.toolchain.errors),
Samuel GAIST's avatar
Samuel GAIST committed
95
96
    )
    nose.tools.eq_(experiment.toolchain.name, "user/integers_addition/2")
André Anjos's avatar
André Anjos committed
97

Philip ABBET's avatar
Philip ABBET committed
98
    nose.tools.eq_(len(experiment.datasets), 1)
99
    nose.tools.assert_true("integers" in experiment.datasets)
Philip ABBET's avatar
Philip ABBET committed
100
101
102
    nose.tools.eq_(len(experiment.databases), 1)
    nose.tools.eq_(len(experiment.blocks), 2)
    nose.tools.eq_(len(experiment.analyzers), 1)
André Anjos's avatar
André Anjos committed
103

Philip ABBET's avatar
Philip ABBET committed
104
    nose.tools.eq_(len(experiment.algorithms), 2)
105
106
107
108
    nose.tools.assert_true("user/sum/1" in experiment.algorithms)
    nose.tools.assert_true(experiment.algorithms["user/sum/1"].valid)
    nose.tools.assert_true("user/integers_analysis/1" in experiment.algorithms)
    nose.tools.assert_true(experiment.algorithms["user/integers_analysis/1"].valid)
Samuel GAIST's avatar
Samuel GAIST committed
109

André Anjos's avatar
André Anjos committed
110
111
112

def test_load_two_datasets_three_blocks_toolchain():

Samuel GAIST's avatar
Samuel GAIST committed
113
    experiment = Experiment(prefix, "user/integers_addition/3/integers_addition")
114
    nose.tools.assert_false(experiment.valid)
Samuel GAIST's avatar
Samuel GAIST committed
115
    nose.tools.eq_(experiment.label, "user/user/integers_addition/3/integers_addition")
116
117
118
    nose.tools.assert_not_equal(
        experiment.errors[0].find("mismatch in input/output"), -1
    )
André Anjos's avatar
André Anjos committed
119

120
    nose.tools.assert_true(experiment.toolchain.valid)
Samuel GAIST's avatar
Samuel GAIST committed
121
    nose.tools.eq_(experiment.toolchain.name, "user/integers_addition/3")
André Anjos's avatar
André Anjos committed
122

Philip ABBET's avatar
Philip ABBET committed
123
    nose.tools.eq_(len(experiment.datasets), 2)
124
125
    nose.tools.assert_true("integers1" in experiment.datasets)
    nose.tools.assert_true("integers2" in experiment.datasets)
Philip ABBET's avatar
Philip ABBET committed
126
127
128
    nose.tools.eq_(len(experiment.databases), 1)
    nose.tools.eq_(len(experiment.blocks), 3)
    nose.tools.eq_(len(experiment.analyzers), 1)
André Anjos's avatar
André Anjos committed
129

Philip ABBET's avatar
Philip ABBET committed
130
    nose.tools.eq_(len(experiment.algorithms), 2)
131
132
133
134
    nose.tools.assert_true("user/sum/1" in experiment.algorithms)
    nose.tools.assert_true(experiment.algorithms["user/sum/1"].valid)
    nose.tools.assert_true("user/integers_analysis/1" in experiment.algorithms)
    nose.tools.assert_true(experiment.algorithms["user/integers_analysis/1"].valid)
Samuel GAIST's avatar
Samuel GAIST committed
135

André Anjos's avatar
André Anjos committed
136
137
138

def test_no_description():

139
    experiment = Experiment(prefix, "errors/user/single/1/single_error")
140
141
142
143
144
    nose.tools.assert_true(
        experiment.valid, "\n  * %s" % "\n  * ".join(experiment.errors)
    )
    nose.tools.assert_is_none(experiment.description)
    nose.tools.assert_is_none(experiment.documentation)
André Anjos's avatar
André Anjos committed
145

Samuel GAIST's avatar
Samuel GAIST committed
146
    description = "This is my descriptor"
Philip ABBET's avatar
Philip ABBET committed
147
    experiment.description = description
148
    nose.tools.assert_true(isinstance(experiment.description, six.string_types))
Philip ABBET's avatar
Philip ABBET committed
149
    nose.tools.eq_(experiment.description, description)
André Anjos's avatar
André Anjos committed
150
151
152
153
154


@nose.tools.with_setup(teardown=cleanup)
def test_export():

Samuel GAIST's avatar
Samuel GAIST committed
155
    name = "user/integers_addition/1/integers_addition"
Philip ABBET's avatar
Philip ABBET committed
156
    obj = Experiment(prefix, name)
157
    nose.tools.assert_true(obj.valid, "\n  * %s" % "\n  * ".join(obj.errors))
André Anjos's avatar
André Anjos committed
158

Philip ABBET's avatar
Philip ABBET committed
159
    obj.export(tmp_prefix)
André Anjos's avatar
André Anjos committed
160

Philip ABBET's avatar
Philip ABBET committed
161
162
    # load from tmp_prefix and validates
    exported = Experiment(tmp_prefix, name)
163
    nose.tools.assert_true(exported.valid, "\n  * %s" % "\n  * ".join(exported.errors))