test_experiments.py 11.3 KB
Newer Older
André Anjos's avatar
André Anjos committed
1
2
3
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

Samuel GAIST's avatar
Samuel GAIST committed
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
###################################################################################
#                                                                                 #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/               #
# Contact: beat.support@idiap.ch                                                  #
#                                                                                 #
# Redistribution and use in source and binary forms, with or without              #
# modification, are permitted provided that the following conditions are met:     #
#                                                                                 #
# 1. Redistributions of source code must retain the above copyright notice, this  #
# list of conditions and the following disclaimer.                                #
#                                                                                 #
# 2. Redistributions in binary form must reproduce the above copyright notice,    #
# this list of conditions and the following disclaimer in the documentation       #
# and/or other materials provided with the distribution.                          #
#                                                                                 #
# 3. Neither the name of the copyright holder nor the names of its contributors   #
# may be used to endorse or promote products derived from this software without   #
# specific prior written permission.                                              #
#                                                                                 #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED   #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE          #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE    #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL      #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR      #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER      #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,   #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE   #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.            #
#                                                                                 #
###################################################################################
André Anjos's avatar
André Anjos committed
35
36
37
38
39


# Basic tests for the command line beat program: experiments

import os
40
import logging
André Anjos's avatar
André Anjos committed
41
import nose.tools
42
43
import click
from click.testing import CliRunner
44
from . import platform, disconnected, prefix, tmp_prefix, user, token
45
from .utils import index_experiment_dbs, MockLoggingHandler
André Anjos's avatar
André Anjos committed
46
from ..common import Selector
47
from beat.cmdline.scripts import main_cli
André Anjos's avatar
André Anjos committed
48
from beat.core.test.utils import slow, cleanup, skipif
49
from beat.core.experiment import Storage, Experiment
50
51
52


def setup_experiments():
53
54
    index_experiment_dbs("user/user/double_triangle/1/double_triangle")
    index_experiment_dbs("user/user/integers_addition/1/integers_addition")
André Anjos's avatar
André Anjos committed
55
56
57


def call(*args, **kwargs):
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
    """A central mechanism to call the main routine with the right parameters"""

    use_prefix = kwargs.get("prefix", prefix)
    use_platform = kwargs.get("platform", platform)
    use_cache = kwargs.get("cache", "cache")

    runner = CliRunner()
    with runner.isolated_filesystem():
        result = runner.invoke(
            main_cli.main,
            [
                "--test-mode",
                "--prefix",
                use_prefix,
                "--token",
                token,
                "--user",
                user,
                "--platform",
                use_platform,
                "--cache",
                use_cache,
                "experiments",
            ]
            + list(args),
83
            catch_exceptions=False,
84
85
86
87
        )
    if result.exit_code != 0:
        click.echo(result.output)
    return result.exit_code
André Anjos's avatar
André Anjos committed
88
89
90
91
92
93


@slow
@nose.tools.with_setup(teardown=cleanup)
@skipif(disconnected, "missing test platform (%s)" % platform)
def test_remote_list():
94
    nose.tools.eq_(call("list", "--remote"), 0)
André Anjos's avatar
André Anjos committed
95
96
97
98
99
100


@slow
@nose.tools.with_setup(teardown=cleanup)
@skipif(disconnected, "missing test platform (%s)" % platform)
def test_pull_one(obj=None):
101
102
103
104
105
    obj = obj or "user/user/single/1/single"
    nose.tools.eq_(call("pull", obj, prefix=tmp_prefix), 0)
    s = Storage(tmp_prefix, obj)
    nose.tools.assert_true(s.exists())
    return s
André Anjos's avatar
André Anjos committed
106
107
108
109
110
111


@slow
@nose.tools.with_setup(teardown=cleanup)
@skipif(disconnected, "missing test platform (%s)" % platform)
def test_pull_all():
112
    nose.tools.eq_(call("pull", prefix=tmp_prefix), 0)
André Anjos's avatar
André Anjos committed
113
114
115
116
117
118


@slow
@nose.tools.with_setup(teardown=cleanup)
@skipif(disconnected, "missing test platform (%s)" % platform)
def test_diff():
119
120
    obj = "user/user/single/1/single"
    nose.tools.eq_(call("pull", obj, prefix=tmp_prefix), 0)
André Anjos's avatar
André Anjos committed
121

122
123
    s = Storage(tmp_prefix, obj)
    nose.tools.assert_true(s.exists())
André Anjos's avatar
André Anjos committed
124

125
126
127
128
    # quickly modify the user experiment:
    f = Experiment(tmp_prefix, obj)
    f.data["globals"]["queue"] = "another_queue"
    f.write()
André Anjos's avatar
André Anjos committed
129

130
    nose.tools.eq_(call("diff", obj, prefix=tmp_prefix), 0)
André Anjos's avatar
André Anjos committed
131
132
133
134
135
136


@slow
@nose.tools.with_setup(teardown=cleanup)
@skipif(disconnected, "missing test platform (%s)" % platform)
def test_status():
137
138
139
    test_diff()
    test_pull_one()
    nose.tools.eq_(call("status", prefix=tmp_prefix), 0)
André Anjos's avatar
André Anjos committed
140
141
142


def test_check_valid():
143
144
    obj = "user/user/single/1/single"
    nose.tools.eq_(call("check", obj), 0)
André Anjos's avatar
André Anjos committed
145
146
147


def test_check_invalid():
148
149
    obj = "user/user/single/1/does_not_exist"
    nose.tools.eq_(call("check", obj), 1)
André Anjos's avatar
André Anjos committed
150
151
152
153
154


@nose.tools.with_setup(teardown=cleanup)
@skipif(disconnected, "missing test platform (%s)" % platform)
def test_fork(obj=None, obj2=None):
155
156
157
158
159
160
    obj = obj or "user/user/single/1/single"
    test_pull_one(obj)
    obj2 = obj2 or "user/user/single/1/different"
    nose.tools.eq_(call("fork", obj, obj2, prefix=tmp_prefix), 0)
    s = Storage(tmp_prefix, obj2)
    nose.tools.assert_true(s.exists())
André Anjos's avatar
André Anjos committed
161

162
163
164
    # check fork status
    with Selector(tmp_prefix) as selector:
        nose.tools.eq_(selector.forked_from("experiment", obj2), obj)
André Anjos's avatar
André Anjos committed
165
166
167
168
169


@nose.tools.with_setup(teardown=cleanup)
@skipif(disconnected, "missing test platform (%s)" % platform)
def test_delete_local():
170
171
172
173
    obj = "user/user/single/1/single"
    storage = test_pull_one(obj)
    nose.tools.eq_(call("rm", obj, prefix=tmp_prefix), 0)
    nose.tools.assert_false(storage.exists())
André Anjos's avatar
André Anjos committed
174
175
176


@slow
177
@nose.tools.with_setup(setup=setup_experiments, teardown=cleanup)
André Anjos's avatar
André Anjos committed
178
def test_run_integers_addition_1():
179
180
    obj = "user/user/integers_addition/1/integers_addition"
    nose.tools.eq_(call("run", obj, cache=tmp_prefix), 0)
André Anjos's avatar
André Anjos committed
181
182
183


@slow
184
@nose.tools.with_setup(setup=setup_experiments, teardown=cleanup)
André Anjos's avatar
André Anjos committed
185
def test_list_integers_addition_1_cache():
186
187
188
    obj = "user/user/integers_addition/1/integers_addition"
    nose.tools.eq_(call("run", obj, cache=tmp_prefix), 0)
    nose.tools.eq_(call("caches", "--list", obj, cache=tmp_prefix), 0)
André Anjos's avatar
André Anjos committed
189
190
191


@slow
192
@nose.tools.with_setup(setup=setup_experiments, teardown=cleanup)
André Anjos's avatar
André Anjos committed
193
def test_checksum_integers_addition_1_cache():
194
195
196
    obj = "user/user/integers_addition/1/integers_addition"
    nose.tools.eq_(call("run", obj, cache=tmp_prefix), 0)
    nose.tools.eq_(call("caches", "--checksum", obj, cache=tmp_prefix), 0)
André Anjos's avatar
André Anjos committed
197
198
199


@slow
200
@nose.tools.with_setup(setup=setup_experiments, teardown=cleanup)
André Anjos's avatar
André Anjos committed
201
def test_delete_integers_addition_1_cache():
202
203
204
    obj = "user/user/integers_addition/1/integers_addition"
    nose.tools.eq_(call("run", obj, cache=tmp_prefix), 0)
    nose.tools.eq_(call("caches", "--delete", obj, cache=tmp_prefix), 0)
André Anjos's avatar
André Anjos committed
205
206


207
208
209
@slow
@nose.tools.with_setup(setup=setup_experiments, teardown=cleanup)
def test_run_integers_addition_1_twice():
210
211
212
    log_handler = MockLoggingHandler(level="DEBUG")
    logging.getLogger().addHandler(log_handler)
    log_messages = log_handler.messages
213

214
215
216
217
218
219
220
221
222
    obj = "user/user/integers_addition/1/integers_addition"
    nose.tools.eq_(call("run", obj, cache=tmp_prefix), 0)
    info_len = len(log_messages["info"])
    nose.tools.assert_greater(info_len, 4)
    nose.tools.assert_true(log_messages["info"][info_len - 1].startswith("  Results"))
    nose.tools.eq_(call("run", obj, cache=tmp_prefix), 0)
    info_len = len(log_messages["info"])
    nose.tools.assert_greater(info_len, 6)
    nose.tools.assert_true(log_messages["info"][info_len - 1].startswith("  Results"))
223
224


André Anjos's avatar
André Anjos committed
225
@slow
226
@nose.tools.with_setup(setup=setup_experiments, teardown=cleanup)
André Anjos's avatar
André Anjos committed
227
def test_run_double_triangle_1():
228
229
    obj = "user/user/double_triangle/1/double_triangle"
    nose.tools.eq_(call("run", obj, cache=tmp_prefix), 0)
André Anjos's avatar
André Anjos committed
230
231
232


@slow
233
@nose.tools.with_setup(setup=setup_experiments, teardown=cleanup)
234
def test_run_single_error_1_local():
235
236
237
238
    # When running locally, the module with the error is loaded
    # inside the currently running process and will return '1'.
    obj = "user/user/single/1/single_error"
    nose.tools.eq_(call("run", obj, "--local", cache=tmp_prefix), 1)
André Anjos's avatar
André Anjos committed
239
240


241
@slow
242
@nose.tools.with_setup(setup=setup_experiments, teardown=cleanup)
243
def test_run_single_error_twice_local():
244
245
246
247
248
    # This one makes sure our output reset is working properly. Both tries should
    # give out the same error.
    obj = "user/user/single/1/single_error"
    nose.tools.eq_(call("run", obj, "--local", cache=tmp_prefix), 1)
    nose.tools.eq_(call("run", obj, "--local", cache=tmp_prefix), 1)
249
250


André Anjos's avatar
André Anjos committed
251
252
253
@nose.tools.with_setup(teardown=cleanup)
@skipif(disconnected, "missing test platform (%s)" % platform)
def test_push_and_delete():
254
255
256
    obj = "user/user/single/1/single"
    obj2 = "user/user/single/1/different"
    test_fork(obj, obj2)
André Anjos's avatar
André Anjos committed
257

258
259
260
    # now push the fork and then delete it remotely
    nose.tools.eq_(call("push", obj2, prefix=tmp_prefix), 0)
    nose.tools.eq_(call("rm", "--remote", obj2, prefix=tmp_prefix), 0)
André Anjos's avatar
André Anjos committed
261
262
263
264
265


@nose.tools.with_setup(teardown=cleanup)
@skipif(disconnected, "missing test platform (%s)" % platform)
def test_draw():
266
267
    obj = "user/user/double_triangle/1/double_triangle"
    test_pull_one(obj)
André Anjos's avatar
André Anjos committed
268

269
270
    # now push the new object and then delete it remotely
    nose.tools.eq_(call("draw", "--path=%s" % tmp_prefix, prefix=tmp_prefix), 0)
André Anjos's avatar
André Anjos committed
271

272
273
274
275
276
277
    nose.tools.assert_true(
        os.path.exists(os.path.join(tmp_prefix, "experiments", obj + ".dot"))
    )
    nose.tools.assert_true(
        os.path.exists(os.path.join(tmp_prefix, "experiments", obj + ".png"))
    )
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309


@slow
@nose.tools.with_setup(teardown=cleanup)
@skipif(disconnected, "missing test platform (%s)" % platform)
def test_start():
    obj = "user/user/double_triangle/1/double_triangle"
    nose.tools.eq_(call("start", obj), 0)


@slow
@nose.tools.with_setup(teardown=cleanup)
@skipif(disconnected, "missing test platform (%s)" % platform)
def test_cancel():
    obj = "user/user/double_triangle/1/double_triangle"
    nose.tools.eq_(call("cancel", obj), 0)


@slow
@nose.tools.with_setup(teardown=cleanup)
@skipif(disconnected, "missing test platform (%s)" % platform)
def test_reset():
    obj = "user/user/double_triangle/1/double_triangle"
    nose.tools.eq_(call("reset", obj), 0)


@slow
@nose.tools.with_setup(teardown=cleanup)
@skipif(disconnected, "missing test platform (%s)" % platform)
def test_runstatus():
    obj = "user/user/double_triangle/1/double_triangle"
    nose.tools.eq_(call("runstatus", obj), 0)