test_docker_execution.py 8.19 KB
Newer Older
1
2
3
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

Samuel GAIST's avatar
Samuel GAIST committed
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
###################################################################################
#                                                                                 #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/               #
# Contact: beat.support@idiap.ch                                                  #
#                                                                                 #
# Redistribution and use in source and binary forms, with or without              #
# modification, are permitted provided that the following conditions are met:     #
#                                                                                 #
# 1. Redistributions of source code must retain the above copyright notice, this  #
# list of conditions and the following disclaimer.                                #
#                                                                                 #
# 2. Redistributions in binary form must reproduce the above copyright notice,    #
# this list of conditions and the following disclaimer in the documentation       #
# and/or other materials provided with the distribution.                          #
#                                                                                 #
# 3. Neither the name of the copyright holder nor the names of its contributors   #
# may be used to endorse or promote products derived from this software without   #
# specific prior written permission.                                              #
#                                                                                 #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED   #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE          #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE    #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL      #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR      #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER      #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,   #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE   #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.            #
#                                                                                 #
###################################################################################
35

36
37

# Tests for experiment execution within Docker containers
38

39
import os
40
import subprocess
41

42
from ..dock import Host
43
44
from ..execution import DockerExecutor

45
from .utils import cleanup
46
from .utils import slow
47
from .utils import skipif
48

49
from .test_execution import BaseExecutionMixIn
50

51
from . import network_name
52
from . import prefix_folder
53
from . import DOCKER_NETWORK_TEST_ENABLED
54

55
from .utils import DOCKER_TEST_IMAGES
Samuel GAIST's avatar
Samuel GAIST committed
56

57
BUILDER_CONTAINER_NAME = "docker.idiap.ch/beat/beat.env.client"
Samuel GAIST's avatar
Samuel GAIST committed
58
59
60
BUILDER_IMAGE = (
    BUILDER_CONTAINER_NAME + ":" + DOCKER_TEST_IMAGES[BUILDER_CONTAINER_NAME]
)
61

Samuel GAIST's avatar
Samuel GAIST committed
62
# ----------------------------------------------------------
63
64


65
class TestDockerExecution(BaseExecutionMixIn):
66
    @classmethod
67
    def setup_class(cls):
68
69
70
        cls.host = Host(raise_on_errors=False)

    @classmethod
71
    def teardown_class(cls):
72
73
        cls.host.teardown()
        cleanup()
74

75
    def teardown(self):
76
77
        self.host.teardown()

Samuel GAIST's avatar
Samuel GAIST committed
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
    def create_executor(
        self,
        prefix,
        configuration,
        tmp_prefix,
        dataformat_cache,
        database_cache,
        algorithm_cache,
    ):
        executor = DockerExecutor(
            self.host,
            prefix,
            configuration,
            tmp_prefix,
            dataformat_cache,
            database_cache,
            algorithm_cache,
        )
96
97
98
99

        executor.debug = os.environ.get("DOCKER_TEST_DEBUG", False) == "True"
        return executor

100
101
    def build_algorithm(self, algorithm):
        test_folder = os.path.abspath(os.path.join(os.path.dirname(__file__)))
Samuel GAIST's avatar
Samuel GAIST committed
102
        scripts_folder = os.path.abspath(os.path.join(test_folder, "scripts"))
103
        sources_folder = os.path.abspath(os.path.join(test_folder, algorithm))
Samuel GAIST's avatar
Samuel GAIST committed
104
        cmd = ["/build.sh"]
105
        builder_container = self.host.create_container(BUILDER_IMAGE, cmd)
106
        builder_container.add_volume("%s/build.sh" % scripts_folder, "/build.sh")
107
108
109
110
111
112
113
        builder_container.add_volume(sources_folder, "/sources", read_only=False)
        builder_container.uid = os.getuid()
        builder_container.set_workdir("/sources")
        builder_container.set_entrypoint("bash")

        self.host.start(builder_container)
        status = self.host.wait(builder_container)
114
115
116
        if status != 0:
            print(self.host.logs(builder_container))

117
118
119
        self.host.rm(builder_container)
        assert status == 0

120
        # Update the tmp prefix with the latest content
Samuel GAIST's avatar
Samuel GAIST committed
121
122
123
124
125
126
127
128
129
130
        subprocess.check_call(
            [
                "rsync",
                "-arz",
                '--exclude="*"',
                '--include="*.so"',
                os.path.join(test_folder, "prefix"),
                prefix_folder,
            ]
        )
131

132
    @slow
133
    @skipif(not DOCKER_NETWORK_TEST_ENABLED, "Network test disabled")
134
    def test_custom_network(self):
Samuel GAIST's avatar
Samuel GAIST committed
135
136
137
138
139
        result = self.execute(
            "user/user/integers_addition/1/integers_addition",
            [{"sum": 495, "nb": 9}],
            network_name=network_name,
        )
140
141

        assert result is None
142

143
144
    @slow
    def test_custom_port_range(self):
Samuel GAIST's avatar
Samuel GAIST committed
145
146
147
148
149
        result = self.execute(
            "user/user/integers_addition/1/integers_addition",
            [{"sum": 495, "nb": 9}],
            port_range="50000:50100",
        )
150
151
152

        assert result is None

153
154
    @slow
    def test_single_1_prepare_error(self):
Samuel GAIST's avatar
Samuel GAIST committed
155
        result = self.execute("user/user/single/1/prepare_error", [None])
156

Samuel GAIST's avatar
Samuel GAIST committed
157
158
        assert result["status"] == 1
        assert result["user_error"] == "'Could not prepare algorithm (returned False)'"
159

160
161
    @slow
    def test_single_1_setup_error(self):
Samuel GAIST's avatar
Samuel GAIST committed
162
        result = self.execute("user/user/single/1/setup_error", [None])
163

Samuel GAIST's avatar
Samuel GAIST committed
164
165
        assert result["status"] == 1
        assert result["user_error"] == "'Could not setup algorithm (returned False)'"
166

167
168
169
170
    # NOT COMPATIBLE YET WITH THE NEW API
    # @slow
    # def test_cxx_double_1(self):
    #     assert self.execute('user/user/double/1/cxx_double', [{'out_data': 42}]) is None
171
172
173
174

    @slow
    def test_cxx_double_legacy(self):
        datasets_uid = os.getuid()
175
176
        self.build_algorithm("prefix/algorithms/user/cxx_integers_echo_legacy")

Samuel GAIST's avatar
Samuel GAIST committed
177
178
179
180
181
        result = self.execute(
            "user/user/double/1/cxx_double_legacy",
            [{"out_data": 42}],
            datasets_uid=datasets_uid,
        )
182
183
184
185
186
        assert result is None

    @slow
    def test_cxx_double_sequential(self):
        datasets_uid = os.getuid()
187
188
        self.build_algorithm("prefix/algorithms/user/cxx_integers_echo_sequential")

Samuel GAIST's avatar
Samuel GAIST committed
189
190
191
192
193
194
195
196
        assert (
            self.execute(
                "user/user/double/1/cxx_double_sequential",
                [{"out_data": 42}],
                datasets_uid=datasets_uid,
            )
            is None
        )
197
198
199
200

    @slow
    def test_cxx_double_autonomous(self):
        datasets_uid = os.getuid()
201
202
        self.build_algorithm("prefix/algorithms/user/cxx_integers_echo_autonomous")

Samuel GAIST's avatar
Samuel GAIST committed
203
204
205
206
207
208
209
210
        assert (
            self.execute(
                "user/user/double/1/cxx_double_autonomous",
                [{"out_data": 42}],
                datasets_uid=datasets_uid,
            )
            is None
        )
211
212
213
214

    @slow
    def test_cxx_analyzer_error(self):
        datasets_uid = os.getuid()
215
216
        needed_alorithms = [
            "cxx_integers_echo_sequential",
Samuel GAIST's avatar
Samuel GAIST committed
217
            "cxx_integers_echo_analyzer",
218
219
220
221
222
        ]

        for algorithm in needed_alorithms:
            self.build_algorithm("prefix/algorithms/user/%s" % algorithm)

Samuel GAIST's avatar
Samuel GAIST committed
223
224
225
226
227
        result = self.execute(
            "user/user/double/1/cxx_analyzer_error",
            [{"out_data": 42}],
            datasets_uid=datasets_uid,
        )
228

Samuel GAIST's avatar
Samuel GAIST committed
229
230
        assert result["status"] == 255
        assert "[sys] C++ algorithm can't be analyzers" in result["stderr"]