test_execution.py 19.3 KB
Newer Older
André Anjos's avatar
André Anjos committed
1
2
3
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

Samuel GAIST's avatar
Samuel GAIST committed
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
###################################################################################
#                                                                                 #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/               #
# Contact: beat.support@idiap.ch                                                  #
#                                                                                 #
# Redistribution and use in source and binary forms, with or without              #
# modification, are permitted provided that the following conditions are met:     #
#                                                                                 #
# 1. Redistributions of source code must retain the above copyright notice, this  #
# list of conditions and the following disclaimer.                                #
#                                                                                 #
# 2. Redistributions in binary form must reproduce the above copyright notice,    #
# this list of conditions and the following disclaimer in the documentation       #
# and/or other materials provided with the distribution.                          #
#                                                                                 #
# 3. Neither the name of the copyright holder nor the names of its contributors   #
# may be used to endorse or promote products derived from this software without   #
# specific prior written permission.                                              #
#                                                                                 #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED   #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE          #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE    #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL      #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR      #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER      #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,   #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE   #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.            #
#                                                                                 #
###################################################################################
André Anjos's avatar
André Anjos committed
35
36
37
38
39
40


# Tests for experiment execution

import glob
import logging
Samuel GAIST's avatar
Samuel GAIST committed
41
import os
42
import subprocess as sp  # nosec
Samuel GAIST's avatar
Samuel GAIST committed
43

44
from shutil import rmtree
André Anjos's avatar
André Anjos committed
45

Samuel GAIST's avatar
Samuel GAIST committed
46
47
48
import nose.tools

from ..data import CachedDataSource
49
from ..execution import LocalExecutor
50
from ..execution import SubprocessExecutor
Samuel GAIST's avatar
Samuel GAIST committed
51
from ..experiment import Experiment
52
from ..hash import hashDataset
Samuel GAIST's avatar
Samuel GAIST committed
53
from ..hash import hashFileContents
54
from ..hash import toPath
Samuel GAIST's avatar
Samuel GAIST committed
55
56
from . import prefix
from . import tmp_prefix
Philip ABBET's avatar
Philip ABBET committed
57
from .utils import slow
58

59
logger = logging.getLogger(__name__)
60

61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93

# ----------------------------------------------------------
# Helpers


def create_conda_environment(additional_packages=[]):
    environment_name = "subprocess_environment"
    environment_prefix = os.path.join(tmp_prefix, environment_name)
    packages = ["python=3"] + additional_packages
    sp.run(
        [
            "conda",
            "create",
            "-y",
            "-c",
            "defaults",
            "-c",
            "http://www.idiap.ch/software/bob/conda/",
            "--prefix",
            environment_prefix,
        ]
        + packages,
        check=True,
        stdout=sp.PIPE,
        stderr=sp.PIPE,
    )
    return environment_prefix


def clear_conda_environment(environment_prefix):
    rmtree(environment_prefix)


Samuel GAIST's avatar
Samuel GAIST committed
94
# ----------------------------------------------------------
95

96

Samuel GAIST's avatar
Samuel GAIST committed
97
class BaseExecutionMixIn(object):
Philip ABBET's avatar
Philip ABBET committed
98
    def check_output(self, prefix, path):
Samuel GAIST's avatar
Samuel GAIST committed
99
        """Checks if a given output exists, together with its indexes and checksums"""
Philip ABBET's avatar
Philip ABBET committed
100
101

        finalpath = os.path.join(prefix, path)
Samuel GAIST's avatar
Samuel GAIST committed
102
103
104
105
        datafiles = glob.glob(finalpath + "*.data")
        datachksums = glob.glob(finalpath + "*.data.checksum")
        indexfiles = glob.glob(finalpath + "*.index")
        indexchksums = glob.glob(finalpath + "*.index.checksum")
106

107
        nose.tools.assert_true(datafiles)
108
        nose.tools.eq_(len(datafiles), len(indexfiles))
Philip ABBET's avatar
Philip ABBET committed
109
        for k in datafiles + indexfiles:
Samuel GAIST's avatar
Samuel GAIST committed
110
            checksum_file = k + ".checksum"
111
            nose.tools.assert_true(checksum_file in datachksums + indexchksums)
Philip ABBET's avatar
Philip ABBET committed
112
            stored_checksum = None
Samuel GAIST's avatar
Samuel GAIST committed
113
114
            with open(checksum_file, "rt") as f:
                stored_checksum = f.read().strip()
Philip ABBET's avatar
Philip ABBET committed
115
            current_checksum = hashFileContents(k)
116
            nose.tools.eq_(current_checksum, stored_checksum)
Philip ABBET's avatar
Philip ABBET committed
117
118

    def load_result(self, executor):
Samuel GAIST's avatar
Samuel GAIST committed
119
        """Loads the result of an experiment, in a single go"""
Philip ABBET's avatar
Philip ABBET committed
120
121

        f = CachedDataSource()
122
123
124
125
126
        nose.tools.assert_true(
            f.setup(
                os.path.join(executor.cache, executor.data["result"]["path"] + ".data"),
                executor.prefix,
            )
Samuel GAIST's avatar
Samuel GAIST committed
127
        )
128

129
        data, start, end = f[0]
130
        nose.tools.eq_(start, 0)
131
        nose.tools.assert_true(end >= start)
Philip ABBET's avatar
Philip ABBET committed
132
133
134
        f.close()
        return data

135
    def execute(self, label, expected_result, **kwargs):
Philip ABBET's avatar
Philip ABBET committed
136
137
138
139
140
141
142
143
        """Executes the full experiment, block after block, returning results. If an
        error occurs, returns information about the err'd block. Otherwise, returns
        ``None``.

        This bit of code mimics the scheduler, but does everything on the local
        machine. It borrows some code from the package ``beat.cmdline``.
        """

144
145
        executor_parameters = kwargs.pop("executor_parameters", {})

Philip ABBET's avatar
Philip ABBET committed
146
147
148
149
        dataformat_cache = {}
        database_cache = {}
        algorithm_cache = {}

Samuel GAIST's avatar
Samuel GAIST committed
150
151
152
        experiment = Experiment(
            prefix, label, dataformat_cache, database_cache, algorithm_cache
        )
Philip ABBET's avatar
Philip ABBET committed
153

154
155
156
        nose.tools.assert_true(
            experiment.valid, "\n  * %s" % "\n  * ".join(experiment.errors)
        )
Philip ABBET's avatar
Philip ABBET committed
157

158
        for block_name, infos in experiment.datasets.items():
Samuel GAIST's avatar
Samuel GAIST committed
159
160
161
162
163
            view = infos["database"].view(infos["protocol"], infos["set"])
            filename = toPath(
                hashDataset(infos["database"].name, infos["protocol"], infos["set"]),
                suffix=".db",
            )
164
165
            view.index(os.path.join(tmp_prefix, filename))

Philip ABBET's avatar
Philip ABBET committed
166
167
168
169
170
        scheduled = experiment.setup()

        # can we execute it?
        results = []
        for key, value in scheduled.items():
Samuel GAIST's avatar
Samuel GAIST committed
171
172
173
174
175
176
177
178
179
            configuration = {**value["configuration"], **kwargs}

            executor = self.create_executor(
                prefix,
                configuration,
                tmp_prefix,
                dataformat_cache,
                database_cache,
                algorithm_cache,
180
                **executor_parameters
Samuel GAIST's avatar
Samuel GAIST committed
181
            )
182
183
184
            nose.tools.assert_true(
                executor.valid, "\n  * %s" % "\n  * ".join(executor.errors)
            )
Philip ABBET's avatar
Philip ABBET committed
185
186
187

            with executor:
                result = executor.process(timeout_in_minutes=3)
188
189
190
191
192
193
194
                nose.tools.assert_true(result)
                nose.tools.assert_true("status" in result)
                nose.tools.assert_true("stdout" in result)
                nose.tools.assert_true("stderr" in result)
                nose.tools.assert_true("timed_out" in result)
                nose.tools.assert_true("system_error" in result)
                nose.tools.assert_true("user_error" in result)
Samuel GAIST's avatar
Samuel GAIST committed
195
                if result["status"] != 0:
Samuel GAIST's avatar
Samuel GAIST committed
196
197
198
199
200
201
202
                    logger.warning("status: %i", result["status"])
                    logger.warning(
                        "(eventual) system errors: %s", result["system_error"]
                    )
                    logger.warning("(eventual) user errors: %s", result["user_error"])
                    logger.warning("stdout: %s", result["stdout"])
                    logger.warning("stderr: %s", result["stderr"])
Philip ABBET's avatar
Philip ABBET committed
203
                    return result
Samuel GAIST's avatar
Samuel GAIST committed
204
                if result["system_error"]:
Samuel GAIST's avatar
Samuel GAIST committed
205
                    logger.warning("system errors: %s", result["system_error"])
Philip ABBET's avatar
Philip ABBET committed
206
                    return result
207
                nose.tools.eq_(result["status"], 0)
Philip ABBET's avatar
Philip ABBET committed
208

Samuel GAIST's avatar
Samuel GAIST committed
209
                if "statistics" in result:
210
                    nose.tools.assert_true(isinstance(result["statistics"], dict))
Philip ABBET's avatar
Philip ABBET committed
211
212

            if executor.analysis:
Samuel GAIST's avatar
Samuel GAIST committed
213
                self.check_output(tmp_prefix, executor.data["result"]["path"])
Philip ABBET's avatar
Philip ABBET committed
214
215
                results.append(self.load_result(executor))
            else:
Samuel GAIST's avatar
Samuel GAIST committed
216
217
                for name, details in executor.data["outputs"].items():
                    self.check_output(tmp_prefix, details["path"])
Philip ABBET's avatar
Philip ABBET committed
218
219

        # compares all results
220
        nose.tools.assert_true(results)
221

Philip ABBET's avatar
Philip ABBET committed
222
223
        for k, result in enumerate(results):
            expected = result.__class__()
Samuel GAIST's avatar
Samuel GAIST committed
224
            expected.from_dict(expected_result[k], casting="unsafe")  # defaults=False
Philip ABBET's avatar
Philip ABBET committed
225

226
227
228
            nose.tools.assert_true(
                result.isclose(expected),
                "%r is not close enough to %r" % (result.as_dict(), expected.as_dict()),
Samuel GAIST's avatar
Samuel GAIST committed
229
            )
Philip ABBET's avatar
Philip ABBET committed
230
231
232

    @slow
    def test_integers_addition_1(self):
233
        nose.tools.assert_is_none(
Samuel GAIST's avatar
Samuel GAIST committed
234
235
236
237
238
            self.execute(
                "user/user/integers_addition/1/integers_addition",
                [{"sum": 495, "nb": 9}],
            )
        )
Philip ABBET's avatar
Philip ABBET committed
239
240
241

    @slow
    def test_integers_addition_2(self):
242
        nose.tools.assert_is_none(
Samuel GAIST's avatar
Samuel GAIST committed
243
244
245
246
247
            self.execute(
                "user/user/integers_addition/2/integers_addition",
                [{"sum": 4995, "nb": 9}],
            )
        )
Philip ABBET's avatar
Philip ABBET committed
248
249
250

    @slow
    def test_single_1_single(self):
251
252
253
        nose.tools.assert_is_none(
            self.execute("user/user/single/1/single", [{"out_data": 42}])
        )
Philip ABBET's avatar
Philip ABBET committed
254
255
256

    @slow
    def test_single_1_add(self):
257
258
259
        nose.tools.assert_is_none(
            self.execute("user/user/single/1/single_add", [{"out_data": 43}])
        )
Philip ABBET's avatar
Philip ABBET committed
260

261
262
263
264
265
266
267
268
269
270
271
272
    @slow
    def test_single_1_env_add(self):
        nose.tools.assert_is_none(
            self.execute("user/user/single/1/single_env_add", [{"out_data": 43}])
        )

    @slow
    def test_single_1_env_add_v2(self):
        nose.tools.assert_is_none(
            self.execute("user/user/single/1/single_env_add_v2", [{"out_data": 43}])
        )

Philip ABBET's avatar
Philip ABBET committed
273
274
    @slow
    def test_single_1_add2(self):
275
276
        nose.tools.assert_is_none(
            self.execute("user/user/single/1/single_add2", [{"out_data": 44}])
Samuel GAIST's avatar
Samuel GAIST committed
277
        )
Philip ABBET's avatar
Philip ABBET committed
278
279
280

    @slow
    def test_single_1_error(self):
281
        result = self.execute("errors/user/single/1/single_error", [None])
282
        nose.tools.assert_true(result)
Samuel GAIST's avatar
Samuel GAIST committed
283
        nose.tools.eq_(result["status"], 1)
284
285
        nose.tools.assert_true(result["user_error"])
        nose.tools.assert_true("NameError" in result["user_error"])
Samuel GAIST's avatar
Samuel GAIST committed
286
        nose.tools.eq_(result["system_error"], "")
Philip ABBET's avatar
Philip ABBET committed
287
288
289

    @slow
    def test_single_1_crash(self):
290
        result = self.execute("errors/user/single/1/single_crash", [None])
291
        nose.tools.assert_true(result)
Samuel GAIST's avatar
Samuel GAIST committed
292
        nose.tools.eq_(result["status"], 1)
293
294
        nose.tools.assert_true(result["user_error"])
        nose.tools.assert_true("NameError" in result["user_error"])
Samuel GAIST's avatar
Samuel GAIST committed
295
        nose.tools.eq_(result["system_error"], "")
Philip ABBET's avatar
Philip ABBET committed
296
297

    @slow
298
    def test_single_1_db_crash(self):
299
        result = self.execute("errors/user/single/1/single_db_crash", [None])
300
301
302
303
        nose.tools.assert_true(result)
        nose.tools.assert_not_equal(result["status"], 0)
        nose.tools.assert_true(result["user_error"])
        nose.tools.assert_true("a = b" in result["user_error"])
Samuel GAIST's avatar
Samuel GAIST committed
304
        nose.tools.eq_(result["system_error"], "")
Philip ABBET's avatar
Philip ABBET committed
305
306
307

    @slow
    def test_single_1_large(self):
308
309
        nose.tools.assert_is_none(
            self.execute("user/user/single/1/single_large", [{"out_data": 2.0}])
Samuel GAIST's avatar
Samuel GAIST committed
310
        )
Philip ABBET's avatar
Philip ABBET committed
311
312
313

    @slow
    def test_double_1(self):
314
315
316
        nose.tools.assert_is_none(
            self.execute("user/user/double/1/double", [{"out_data": 42}])
        )
Philip ABBET's avatar
Philip ABBET committed
317
318
319

    @slow
    def test_triangle_1(self):
320
321
322
        nose.tools.assert_is_none(
            self.execute("user/user/triangle/1/triangle", [{"out_data": 42}])
        )
Philip ABBET's avatar
Philip ABBET committed
323
324
325

    @slow
    def test_too_many_nexts(self):
326
        result = self.execute("errors/user/triangle/1/too_many_nexts", [None])
327
328
329
330
        nose.tools.assert_true(result)
        nose.tools.assert_not_equal(result["status"], 0)
        nose.tools.assert_true(result["user_error"])
        nose.tools.assert_true("no more data" in result["user_error"])
Philip ABBET's avatar
Philip ABBET committed
331
332
333

    @slow
    def test_double_triangle_1(self):
334
        nose.tools.assert_is_none(
Samuel GAIST's avatar
Samuel GAIST committed
335
336
337
338
            self.execute(
                "user/user/double_triangle/1/double_triangle", [{"out_data": 42}]
            )
        )
Philip ABBET's avatar
Philip ABBET committed
339
340
341

    @slow
    def test_inputs_mix_1(self):
342
343
        nose.tools.assert_is_none(
            self.execute("user/user/inputs_mix/1/test", [{"sum": 495, "nb": 9}])
Samuel GAIST's avatar
Samuel GAIST committed
344
        )
Philip ABBET's avatar
Philip ABBET committed
345
346
347

    @slow
    def test_inputs_mix_2(self):
348
349
        nose.tools.assert_is_none(
            self.execute("user/user/inputs_mix/2/test", [{"sum": 495, "nb": 9}])
Samuel GAIST's avatar
Samuel GAIST committed
350
        )
Philip ABBET's avatar
Philip ABBET committed
351
352
353

    @slow
    def test_inputs_mix_3(self):
354
355
        nose.tools.assert_is_none(
            self.execute("user/user/inputs_mix/3/test", [{"sum": 945, "nb": 9}])
Samuel GAIST's avatar
Samuel GAIST committed
356
        )
Philip ABBET's avatar
Philip ABBET committed
357
358
359

    @slow
    def test_inputs_mix_3b(self):
360
        nose.tools.assert_is_none(
Samuel GAIST's avatar
Samuel GAIST committed
361
362
            self.execute("user/user/inputs_mix/3/test2", [{"sum": 954, "nb": 9}])
        )
Philip ABBET's avatar
Philip ABBET committed
363
364
365

    @slow
    def test_inputs_mix_4(self):
366
367
        nose.tools.assert_is_none(
            self.execute("user/user/inputs_mix/4/test", [{"sum": 990, "nb": 9}])
Samuel GAIST's avatar
Samuel GAIST committed
368
        )
Philip ABBET's avatar
Philip ABBET committed
369
370
371

    @slow
    def test_inputs_mix_4b(self):
372
        nose.tools.assert_is_none(
Samuel GAIST's avatar
Samuel GAIST committed
373
374
            self.execute("user/user/inputs_mix/4/test2", [{"sum": 1008, "nb": 9}])
        )
Philip ABBET's avatar
Philip ABBET committed
375
376
377

    @slow
    def test_integers_labelled_1(self):
378
        nose.tools.assert_is_none(
Samuel GAIST's avatar
Samuel GAIST committed
379
380
381
382
383
            self.execute(
                "user/user/integers_labelled/1/test",
                [{"nb_data_units": 3, "indices": "0 - 4\n5 - 9\n10 - 14\n"}],
            )
        )
Philip ABBET's avatar
Philip ABBET committed
384

385
386
    @slow
    def test_preprocessing_1(self):
387
        nose.tools.assert_is_none(
Samuel GAIST's avatar
Samuel GAIST committed
388
389
390
391
392
            self.execute(
                "user/user/preprocessing/1/different_frequencies",
                [{"sum": 363, "nb": 8}],
            )
        )
393

394
395
    @slow
    def test_single_1_prepare_success(self):
396
        nose.tools.assert_is_none(
Samuel GAIST's avatar
Samuel GAIST committed
397
398
            self.execute("user/user/single/1/prepare_success", [{"out_data": 42}])
        )
399

400
401
    @slow
    def test_loop_1(self):
402
        nose.tools.assert_is_none(
403
            self.execute(
404
                "user/user/loop/1/loop", [{"sum": 135, "nb": 9}, {"sum": 9, "nb": 9}]
405
            )
406
        )
407

408
409
410
411
412
    @slow
    def test_two_loops_1(self):
        nose.tools.assert_is_none(
            self.execute(
                "user/user/two_loops/1/two_loops",
413
                [{"sum": 621, "nb": 9}, {"sum": 108, "nb": 9}],
414
415
416
            )
        )

417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
    @slow
    def test_single_1_write_in_wrong_output_seq(self):
        result = self.execute("errors/user/single/1/write_in_wrong_output_seq", [None])
        nose.tools.assert_true(result)
        nose.tools.assert_not_equal(result["status"], 0)
        nose.tools.assert_true(result["user_error"])
        error_message = result["user_error"].replace("\\'", "'")
        nose.tools.assert_true(
            "'NoneType' object has no attribute 'write'" in error_message
        )

    @slow
    def test_single_1_write_in_wrong_output_aut(self):
        result = self.execute("errors/user/single/1/write_in_wrong_output_aut", [None])
        nose.tools.assert_true(result)
        nose.tools.assert_not_equal(result["status"], 0)
        nose.tools.assert_true(result["user_error"])
        error_message = result["user_error"].replace("\\'", "'")
        nose.tools.assert_true(
            "'NoneType' object has no attribute 'write'" in error_message
        )

Philip ABBET's avatar
Philip ABBET committed
439
    # For benchmark purposes
440
    # @slow
Philip ABBET's avatar
Philip ABBET committed
441
    # def test_double_1_large(self):
442
443
444
    #     import time
    #     start = time.time()
    #     assert self.execute('user/user/double/1/large', [{'out_data': 49489830}]) is None
445
    #     print(time.time() - start)
Philip ABBET's avatar
Philip ABBET committed
446
447

    # For benchmark purposes
448
    # @slow
Philip ABBET's avatar
Philip ABBET committed
449
    # def test_double_1_large2(self):
450
451
452
    #     import time
    #     start = time.time()
    #     assert self.execute('user/user/double/1/large2', [{'out_data': 21513820}]) is None
453
    #     print(time.time() - start)
454

455

Samuel GAIST's avatar
Samuel GAIST committed
456
# ----------------------------------------------------------
457
458


459
class TestLocalExecution(BaseExecutionMixIn):
Samuel GAIST's avatar
Samuel GAIST committed
460
461
462
463
464
465
466
467
    def create_executor(
        self,
        prefix,
        configuration,
        tmp_prefix,
        dataformat_cache,
        database_cache,
        algorithm_cache,
468
        **kwargs
Samuel GAIST's avatar
Samuel GAIST committed
469
470
471
472
473
474
475
476
477
    ):
        return LocalExecutor(
            prefix,
            configuration,
            tmp_prefix,
            dataformat_cache,
            database_cache,
            algorithm_cache,
        )
478

479
480
481
    @slow
    def test_single_1_prepare_error(self):
        with nose.tools.assert_raises(RuntimeError) as context:
482
            self.execute("errors/user/single/1/prepare_error", [None])
483
            nose.tools.assert_true("Algorithm prepare failed" in context.exception)
484

485
486
487
    @slow
    def test_single_1_setup_error(self):
        with nose.tools.assert_raises(RuntimeError) as context:
488
            self.execute("errors/user/single/1/setup_error", [None])
489
            nose.tools.assert_true("Algorithm setup failed" in context.exception)
490

491

Samuel GAIST's avatar
Samuel GAIST committed
492
# ----------------------------------------------------------
493
494


495
class TestSubprocessExecution(BaseExecutionMixIn):
Samuel GAIST's avatar
Samuel GAIST committed
496
497
498
499
500
501
502
503
    def create_executor(
        self,
        prefix,
        configuration,
        tmp_prefix,
        dataformat_cache,
        database_cache,
        algorithm_cache,
504
        python_path=None,
Samuel GAIST's avatar
Samuel GAIST committed
505
506
    ):
        return SubprocessExecutor(
507
508
509
510
511
512
513
            prefix=prefix,
            data=configuration,
            cache=tmp_prefix,
            dataformat_cache=dataformat_cache,
            database_cache=database_cache,
            algorithm_cache=algorithm_cache,
            python_path=python_path,
Samuel GAIST's avatar
Samuel GAIST committed
514
        )
515
516
517

    @slow
    def test_single_1_prepare_error(self):
518
        result = self.execute("errors/user/single/1/prepare_error", [None])
519

520
521
522
523
        nose.tools.eq_(result["status"], 1)
        nose.tools.eq_(
            result["user_error"], "'Could not prepare algorithm (returned False)'"
        )
524
525
526

    @slow
    def test_single_1_setup_error(self):
527
        result = self.execute("errors/user/single/1/setup_error", [None])
528

529
530
531
532
        nose.tools.eq_(result["status"], 1)
        nose.tools.eq_(
            result["user_error"], "'Could not setup algorithm (returned False)'"
        )
533
534
535

    @slow
    def test_different_environment(self):
536
        environment_prefix = create_conda_environment(["beat.backend.python"])
537
538
539
540
541
542
543
        result = self.execute(
            "user/user/loop/1/loop",
            [{"sum": 135, "nb": 9}, {"sum": 9, "nb": 9}],
            executor_parameters={
                "python_path": os.path.join(environment_prefix, "bin", "python")
            },
        )
544
        clear_conda_environment(environment_prefix)
545
546
547
548
549

        nose.tools.assert_is_none(result)

    @slow
    def test_wrong_different_environment(self):
550
        environment_prefix = create_conda_environment()
551
552
553
554
555
556
557
558
        with nose.tools.assert_raises(RuntimeError):
            self.execute(
                "user/user/loop/1/loop",
                [{"sum": 135, "nb": 9}, {"sum": 9, "nb": 9}],
                executor_parameters={
                    "python_path": os.path.join(environment_prefix, "bin", "python")
                },
            )
559
        clear_conda_environment(environment_prefix)