test_docker_execution.py 11.9 KB
Newer Older
1
2
3
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

Samuel GAIST's avatar
Samuel GAIST committed
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
###################################################################################
#                                                                                 #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/               #
# Contact: beat.support@idiap.ch                                                  #
#                                                                                 #
# Redistribution and use in source and binary forms, with or without              #
# modification, are permitted provided that the following conditions are met:     #
#                                                                                 #
# 1. Redistributions of source code must retain the above copyright notice, this  #
# list of conditions and the following disclaimer.                                #
#                                                                                 #
# 2. Redistributions in binary form must reproduce the above copyright notice,    #
# this list of conditions and the following disclaimer in the documentation       #
# and/or other materials provided with the distribution.                          #
#                                                                                 #
# 3. Neither the name of the copyright holder nor the names of its contributors   #
# may be used to endorse or promote products derived from this software without   #
# specific prior written permission.                                              #
#                                                                                 #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED   #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE          #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE    #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL      #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR      #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER      #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,   #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE   #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.            #
#                                                                                 #
###################################################################################
35

36
37

# Tests for experiment execution within Docker containers
38

39
import os
40
import subprocess  # nosec
Samuel GAIST's avatar
Samuel GAIST committed
41

42
import nose.tools
43
44

from beat.core.database import Database
45

46
from ..dock import Host
47
from ..execution import DockerExecutor
Samuel GAIST's avatar
Samuel GAIST committed
48
from . import DOCKER_NETWORK_TEST_ENABLED
49
from . import network_name
50
from . import prefix as test_prefix
51
from . import prefix_folder
52
from . import setup_root_db_folder
Samuel GAIST's avatar
Samuel GAIST committed
53
from .test_execution import BaseExecutionMixIn
54
from .utils import DOCKER_TEST_IMAGES
Samuel GAIST's avatar
Samuel GAIST committed
55
56
57
from .utils import cleanup
from .utils import skipif
from .utils import slow
Samuel GAIST's avatar
Samuel GAIST committed
58

59
BUILDER_CONTAINER_NAME = "docker.idiap.ch/beat/beat.env.builder/beat.env.cxxdev"
Samuel GAIST's avatar
Samuel GAIST committed
60
61
62
BUILDER_IMAGE = (
    BUILDER_CONTAINER_NAME + ":" + DOCKER_TEST_IMAGES[BUILDER_CONTAINER_NAME]
)
63

Samuel GAIST's avatar
Samuel GAIST committed
64
# ----------------------------------------------------------
65
66


67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
def write_rawdata_for_database(database_name, raw_data):
    """Generate raw data for give database"""

    db = Database(test_prefix, database_name)
    nose.tools.assert_true(db.valid, db.errors)

    data_sharing_path = db.data["root_folder"]

    with open(os.path.join(data_sharing_path, "datafile.txt"), "wt") as data_file:
        data_file.write("{}".format(raw_data))


# ----------------------------------------------------------


82
class TestDockerExecution(BaseExecutionMixIn):
83
    @classmethod
84
    def setup_class(cls):
85
86
87
        cls.host = Host(raise_on_errors=False)

    @classmethod
88
    def teardown_class(cls):
89
90
        cls.host.teardown()
        cleanup()
91

92
    def teardown(self):
93
94
        self.host.teardown()

Samuel GAIST's avatar
Samuel GAIST committed
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
    def create_executor(
        self,
        prefix,
        configuration,
        tmp_prefix,
        dataformat_cache,
        database_cache,
        algorithm_cache,
    ):
        executor = DockerExecutor(
            self.host,
            prefix,
            configuration,
            tmp_prefix,
            dataformat_cache,
            database_cache,
            algorithm_cache,
        )
113
114
115
116

        executor.debug = os.environ.get("DOCKER_TEST_DEBUG", False) == "True"
        return executor

117
118
    def build_algorithm(self, algorithm):
        test_folder = os.path.abspath(os.path.join(os.path.dirname(__file__)))
Samuel GAIST's avatar
Samuel GAIST committed
119
        scripts_folder = os.path.abspath(os.path.join(test_folder, "scripts"))
120
        sources_folder = os.path.abspath(os.path.join(test_folder, algorithm))
Samuel GAIST's avatar
Samuel GAIST committed
121
        cmd = ["/build.sh"]
122
        builder_container = self.host.create_container(BUILDER_IMAGE, cmd)
123
        builder_container.add_volume("%s/build.sh" % scripts_folder, "/build.sh")
124
125
126
127
128
129
130
        builder_container.add_volume(sources_folder, "/sources", read_only=False)
        builder_container.uid = os.getuid()
        builder_container.set_workdir("/sources")
        builder_container.set_entrypoint("bash")

        self.host.start(builder_container)
        status = self.host.wait(builder_container)
131
132
133
        if status != 0:
            print(self.host.logs(builder_container))

134
        self.host.rm(builder_container)
135
        nose.tools.eq_(status, 0)
136

137
        # Update the tmp prefix with the latest content
138
        subprocess.check_call(  # nosec
Samuel GAIST's avatar
Samuel GAIST committed
139
140
141
142
143
144
145
146
147
            [
                "rsync",
                "-arz",
                '--exclude="*"',
                '--include="*.so"',
                os.path.join(test_folder, "prefix"),
                prefix_folder,
            ]
        )
148

149
150
        setup_root_db_folder()

151
    @slow
152
    @skipif(not DOCKER_NETWORK_TEST_ENABLED, "Network test disabled")
153
    def test_custom_network(self):
Samuel GAIST's avatar
Samuel GAIST committed
154
155
156
157
158
        result = self.execute(
            "user/user/integers_addition/1/integers_addition",
            [{"sum": 495, "nb": 9}],
            network_name=network_name,
        )
159

160
        nose.tools.assert_is_none(result)
161

162
163
    @slow
    def test_custom_port_range(self):
Samuel GAIST's avatar
Samuel GAIST committed
164
165
166
167
168
        result = self.execute(
            "user/user/integers_addition/1/integers_addition",
            [{"sum": 495, "nb": 9}],
            port_range="50000:50100",
        )
169

170
        nose.tools.assert_is_none(result)
171

172
    @slow
173
    def test_database_rawdata_access(self):
174
        offset = 12
175
176

        write_rawdata_for_database("simple_rawdata_access/1", offset)
177
178

        result = self.execute(
179
            "user/user/single/1/single_rawdata_access", [{"out_data": 42 + offset}]
180
181
182
183
        )

        nose.tools.assert_is_none(result)

184
185
186
187
188
189
190
191
192
    @slow
    def test_database_no_rawdata_access(self):
        write_rawdata_for_database("simple/1", "should not be loaded")

        result = self.execute("errors/user/single/1/single_no_rawdata_access", [None])

        nose.tools.eq_(result["status"], 1)
        nose.tools.assert_true("FileNotFoundError" in result["user_error"])

193
194
    @slow
    def test_single_1_prepare_error(self):
195
        result = self.execute("errors/user/single/1/prepare_error", [None])
196

197
198
199
200
        nose.tools.eq_(result["status"], 1)
        nose.tools.eq_(
            result["user_error"], "'Could not prepare algorithm (returned False)'"
        )
201

202
203
    @slow
    def test_single_1_setup_error(self):
204
        result = self.execute("errors/user/single/1/setup_error", [None])
205

206
207
208
209
        nose.tools.eq_(result["status"], 1)
        nose.tools.eq_(
            result["user_error"], "'Could not setup algorithm (returned False)'"
        )
210

211
212
213
214
    # NOT COMPATIBLE YET WITH THE NEW API
    # @slow
    # def test_cxx_double_1(self):
    #     assert self.execute('user/user/double/1/cxx_double', [{'out_data': 42}]) is None
215
216
217
218

    @slow
    def test_cxx_double_legacy(self):
        datasets_uid = os.getuid()
219
220
        self.build_algorithm("prefix/algorithms/user/cxx_integers_echo_legacy")

Samuel GAIST's avatar
Samuel GAIST committed
221
222
223
224
225
        result = self.execute(
            "user/user/double/1/cxx_double_legacy",
            [{"out_data": 42}],
            datasets_uid=datasets_uid,
        )
226
        nose.tools.assert_is_none(result)
227
228
229
230

    @slow
    def test_cxx_double_sequential(self):
        datasets_uid = os.getuid()
231
232
        self.build_algorithm("prefix/algorithms/user/cxx_integers_echo_sequential")

233
        nose.tools.assert_is_none(
Samuel GAIST's avatar
Samuel GAIST committed
234
235
236
237
238
239
            self.execute(
                "user/user/double/1/cxx_double_sequential",
                [{"out_data": 42}],
                datasets_uid=datasets_uid,
            )
        )
240

241
242
243
244
245
246
247
248
249
250
251
252
253
    @slow
    def test_cxx_double_offsetting_sequential(self):
        datasets_uid = os.getuid()
        self.build_algorithm("prefix/algorithms/user/cxx_integers_offsetter_sequential")

        nose.tools.assert_is_none(
            self.execute(
                "user/user/double/1/cxx_offsetting_sequential",
                [{"out_data": 77}],
                datasets_uid=datasets_uid,
            )
        )

254
255
256
    @slow
    def test_cxx_double_autonomous(self):
        datasets_uid = os.getuid()
257
258
        self.build_algorithm("prefix/algorithms/user/cxx_integers_echo_autonomous")

259
        nose.tools.assert_is_none(
Samuel GAIST's avatar
Samuel GAIST committed
260
261
262
263
264
265
            self.execute(
                "user/user/double/1/cxx_double_autonomous",
                [{"out_data": 42}],
                datasets_uid=datasets_uid,
            )
        )
266
267
268
269

    @slow
    def test_cxx_analyzer_error(self):
        datasets_uid = os.getuid()
270
271
        needed_alorithms = [
            "cxx_integers_echo_sequential",
Samuel GAIST's avatar
Samuel GAIST committed
272
            "cxx_integers_echo_analyzer",
273
274
275
276
277
        ]

        for algorithm in needed_alorithms:
            self.build_algorithm("prefix/algorithms/user/%s" % algorithm)

Samuel GAIST's avatar
Samuel GAIST committed
278
        result = self.execute(
279
            "errors/user/double/1/cxx_analyzer_error",
Samuel GAIST's avatar
Samuel GAIST committed
280
281
282
            [{"out_data": 42}],
            datasets_uid=datasets_uid,
        )
283

284
285
286
287
        nose.tools.eq_(result["status"], 255)
        nose.tools.assert_true(
            "[sys] C++ algorithm can't be analyzers" in result["stderr"]
        )
288
289
290
291
292
293

    @slow
    def test_read_only_error(self):
        result = self.execute("errors/user/single/1/write_error", [{"out_data": 42}])

        nose.tools.eq_(result["status"], 1)
294
        nose.tools.assert_true("Read-only" in result["user_error"])
295
296
297
298
299
300
301
302
303

    @slow
    def test_user_mismatch_error(self):
        result = self.execute(
            "errors/user/single/1/write_error", [{"out_data": 42}], datasets_uid=0
        )

        nose.tools.eq_(result["status"], 1)
        nose.tools.assert_true("Failed to create an user" in result["stderr"])
304
305
306
307
308

    @slow
    def test_loop_mix_db_env_error(self):
        with nose.tools.assert_raises(RuntimeError) as context:
            self.execute(
Samuel GAIST's avatar
Samuel GAIST committed
309
310
                "errors/user/loop/1/loop_mix_db_env",
                [None],
311
312
313
314
315
316
317
318
319
320
            )

        nose.tools.assert_true(
            "are not all providing an environment" in context.exception.args[0]
        )

    @slow
    def test_loop_two_db_env_error(self):
        with nose.tools.assert_raises(RuntimeError) as context:
            self.execute(
Samuel GAIST's avatar
Samuel GAIST committed
321
322
                "errors/user/loop/1/loop_two_db_environments",
                [None],
323
324
325
326
327
328
329
330
331
332
            )

        nose.tools.assert_true(
            "are requesting different environments" in context.exception.args[0]
        )

    @slow
    def test_single_not_existing_db_env_error(self):
        with nose.tools.assert_raises(RuntimeError) as context:
            self.execute(
Samuel GAIST's avatar
Samuel GAIST committed
333
334
                "errors/user/single/1/not_existing_db_env",
                [None],
335
336
337
338
339
340
341
342
343
344
345
346
347
348
            )

        nose.tools.assert_true(
            "not found - available environments are" in context.exception.args[0]
        )

    @slow
    def test_loop_1_two_db_env(self):
        nose.tools.assert_is_none(
            self.execute(
                "user/user/loop/1/loop_two_db_env",
                [{"sum": 135, "nb": 9}, {"sum": 9, "nb": 9}],
            )
        )