test_execution.py 11.4 KB
Newer Older
André Anjos's avatar
André Anjos committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

###############################################################################
#                                                                             #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
# Contact: beat.support@idiap.ch                                              #
#                                                                             #
# This file is part of the beat.core module of the BEAT platform.             #
#                                                                             #
# Commercial License Usage                                                    #
# Licensees holding valid commercial BEAT licenses may use this file in       #
# accordance with the terms contained in a written agreement between you      #
# and Idiap. For further information contact tto@idiap.ch                     #
#                                                                             #
# Alternatively, this file may be used under the terms of the GNU Affero      #
# Public License version 3 as published by the Free Software and appearing    #
# in the file LICENSE.AGPL included in the packaging of this file.            #
# The BEAT platform is distributed in the hope that it will be useful, but    #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
#                                                                             #
# You should have received a copy of the GNU Affero Public License along      #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
#                                                                             #
###############################################################################


# Tests for experiment execution

import os
import glob
import logging
logger = logging.getLogger(__name__)

# in case you want to see the printouts dynamically, set to ``True``
if False:
  logger = logging.getLogger() #root logger
  logger.setLevel(logging.DEBUG)
  ch = logging.StreamHandler()
  ch.setLevel(logging.DEBUG)
  ch.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
  logger.addHandler(ch)

import numpy
46
import unittest
André Anjos's avatar
André Anjos committed
47
48

from ..experiment import Experiment
49
from ..execution import DockerExecutor
50
from ..execution import LocalExecutor
André Anjos's avatar
André Anjos committed
51
52
from ..hash import hashFileContents
from ..data import CachedDataSource
53
from ..dock import Host
André Anjos's avatar
André Anjos committed
54
55

from . import prefix, tmp_prefix
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
from .utils import cleanup


class TestExecution(unittest.TestCase):

  def check_output(self, prefix, path):
    '''Checks if a given output exists, together with its indexes and checksums
    '''

    finalpath = os.path.join(prefix, path)
    datafiles = glob.glob(finalpath + '*.data')
    datachksums = glob.glob(finalpath + '*.data.checksum')
    indexfiles = glob.glob(finalpath + '*.index')
    indexchksums = glob.glob(finalpath + '*.index.checksum')
    assert datafiles
    self.assertEqual(len(datafiles), len(indexfiles))
    for k in datafiles + indexfiles:
      checksum_file = k + '.checksum'
      assert checksum_file in datachksums + indexchksums
      stored_checksum = None
      with open(checksum_file, 'rt') as f: stored_checksum = f.read().strip()
      current_checksum = hashFileContents(k)
      self.assertEqual(current_checksum, stored_checksum)


  def load_result(self, executor):
    '''Loads the result of an experiment, in a single go'''

    f = CachedDataSource()
    assert f.setup(os.path.join(executor.cache,
        executor.data['result']['path'] + '.data'), executor.prefix)
    data, start, end = f.next()
    self.assertEqual(start, 0)
    self.assertEqual(start, end)
    f.close()
    return data


  def execute(self, label, expected_result):
    """Executes the full experiment, block after block, returning results. If an
    error occurs, returns information about the err'd block. Otherwise, returns
    ``None``.

    This bit of code mimics the scheduler, but does everything on the local
    machine. It borrows some code from the package ``beat.cmdline``.
    """

    dataformat_cache = {}
    database_cache = {}
    algorithm_cache = {}

    experiment = Experiment(prefix, label,
            dataformat_cache, database_cache, algorithm_cache)

    assert experiment.valid, '\n  * %s' % '\n  * '.join(experiment.errors)

    scheduled = experiment.setup()

    # can we execute it?
    results = []
    for key, value in scheduled.items():
117
118
      executor = self.create_executor(prefix, value['configuration'], tmp_prefix,
                                      dataformat_cache, database_cache, algorithm_cache)
119
120
121
      assert executor.valid, '\n  * %s' % '\n  * '.join(executor.errors)

      with executor:
122
        result = executor.process(timeout_in_minutes=3)
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
        assert result
        assert 'status' in result
        assert 'stdout' in result
        assert 'stderr' in result
        assert 'timed_out' in result
        assert 'system_error' in result
        assert 'user_error' in result
        if result['status'] != 0:
          logger.warn("(eventual) system errors: %s", result['system_error'])
          logger.warn("(eventual) user errors: %s", result['user_error'])
          logger.warn("stdout: %s", result['stdout'])
          logger.warn("stderr: %s", result['stderr'])
          return result
        if result['system_error']:
          logger.warn("system errors: %s", result['system_error'])
          return result
        assert result['status'] == 0
140
141
142

        if result.has_key('statistics'):
          assert isinstance(result['statistics'], dict)
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183

      if executor.analysis:
        self.check_output(tmp_prefix, executor.data['result']['path'])
        results.append(self.load_result(executor))
      else:
        for name, details in executor.data['outputs'].items():
          self.check_output(tmp_prefix, details['path'])

    # compares all results
    assert results
    for k, result in enumerate(results):
      expected = result.__class__()
      expected.from_dict(expected_result[k], casting='unsafe') #defaults=False
      assert result.isclose(expected), "%r is not close enough to %r" % (result.as_dict(), expected.as_dict())


  def test_integers_addition_1(self):
    assert self.execute('user/user/integers_addition/1/integers_addition',
            [{'sum': 12272, 'nb': 10}]) is None

  def test_integers_addition_2(self):
    assert self.execute('user/user/integers_addition/2/integers_addition',
            [{'sum': 18392, 'nb': 10}]) is None

  def test_single_1_single(self):
    assert self.execute('user/user/single/1/single', [{'out_data': 42}]) is None

  def test_single_1_add(self):
    assert self.execute('user/user/single/1/single_add', [{'out_data': 43}]) is None

  def test_single_1_add2(self):
    assert self.execute('user/user/single/1/single_add2', [{'out_data': 44}]) is None

  def test_single_1_error(self):
    result = self.execute('user/user/single/1/single_error', [None])
    assert result
    self.assertEqual(result['status'], 1)
    assert result['user_error']
    assert 'NameError' in result['user_error']
    self.assertEqual(result['system_error'], '')

184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
  def test_single_1_crash(self):
    result = self.execute('user/user/single/1/single_crash', [None])
    assert result
    self.assertEqual(result['status'], 1)
    assert result['user_error']
    assert 'NameError' in result['user_error']
    self.assertEqual(result['system_error'], '')

  def test_single_1_db_crash_done(self):
    result = self.execute('user/user/single/1/single_db_crash_done', [None])
    assert result
    self.assertTrue(result['status'] != 0)
    assert result['user_error']
    assert 'a = b' in result['user_error']
    self.assertEqual(result['system_error'], '')

  def test_single_1_db_crash_next(self):
    result = self.execute('user/user/single/1/single_db_crash_next', [None])
    assert result
    self.assertTrue(result['status'] != 0)
    assert result['user_error']
    assert 'a = b' in result['user_error']
    self.assertEqual(result['system_error'], '')

208
209
210
211
212
213
214
215
216
217
218
219
  def test_single_1_large(self):
    assert self.execute('user/user/single/1/single_large', [{'out_data': 2.0}]) is None

  def test_double_1(self):
    assert self.execute('user/user/double/1/double', [{'out_data': 42}]) is None

  def test_triangle_1(self):
    assert self.execute('user/user/triangle/1/triangle', [{'out_data': 42}]) is None

  def test_too_many_nexts(self):
    result = self.execute('user/user/triangle/1/too_many_nexts', [None])
    assert result
220
    self.assertTrue(result['status'] != 0)
221
222
223
224
225
    assert result['user_error']
    assert 'no more data' in result['user_error']

  def test_double_triangle_1(self):
    assert self.execute('user/user/double_triangle/1/double_triangle', [{'out_data': 42}]) is None
Philip ABBET's avatar
Philip ABBET committed
226

227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
  def test_inputs_mix_1(self):
    assert self.execute('user/user/inputs_mix/1/test', [{'sum': 12272, 'nb': 10}]) is None

  def test_inputs_mix_2(self):
    assert self.execute('user/user/inputs_mix/2/test', [{'sum': 13529, 'nb': 10}]) is None

  def test_inputs_mix_3(self):
    assert self.execute('user/user/inputs_mix/3/test', [{'sum': 19523, 'nb': 10}]) is None

  def test_inputs_mix_3b(self):
    assert self.execute('user/user/inputs_mix/3/test2', [{'sum': 19533, 'nb': 10}]) is None

  def test_inputs_mix_4(self):
    assert self.execute('user/user/inputs_mix/4/test', [{'sum': 25150, 'nb': 10}]) is None

  def test_inputs_mix_4b(self):
    assert self.execute('user/user/inputs_mix/4/test2', [{'sum': 25170, 'nb': 10}]) is None
244
245
246
247
248
249
250
251

  def test_integers_labelled_1(self):
    assert self.execute('user/user/integers_labelled/1/test', [
        {
            'nb_data_units': 3,
            'indices': '0 - 4\n5 - 9\n10 - 14\n'
        }
    ]) is None
252

253
254
255
256
257
258
259
260
261
262
263
264
265
266
  # For benchmark purposes
  # def test_double_1_large(self):
  #   import time
  #   start = time.time()
  #   assert self.execute('user/user/double/1/large', [{'out_data': 49489830}]) is None
  #   print time.time() - start

  # For benchmark purposes
  # def test_double_1_large2(self):
  #   import time
  #   start = time.time()
  #   assert self.execute('user/user/double/1/large2', [{'out_data': 21513820}]) is None
  #   print time.time() - start

267
268


269
270
271
272
273
274
275
276
277
278
279
280
281
class TestDockerExecution(TestExecution):

  @classmethod
  def setUpClass(cls):
    cls.host = Host()
    cls.host.setup(raise_on_errors=False)


  @classmethod
  def tearDownClass(cls):
    cls.host.teardown()
    cleanup()

282
283

  def setUp(self):
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
    super(TestDockerExecution, self).setUp()
    self.proxy_mode = True


  def create_executor(self, prefix, configuration, tmp_prefix, dataformat_cache,
                      database_cache, algorithm_cache):
    return DockerExecutor(self.host, prefix, configuration, tmp_prefix,
                          dataformat_cache, database_cache, algorithm_cache,
                          proxy_mode=self.proxy_mode)


  def test_cxx_double_1(self):
    assert self.execute('user/user/double/1/cxx_double', [{'out_data': 42}]) is None




class TestDockerExecutionNoProxy(TestDockerExecution):

  def setUp(self):
    super(TestDockerExecutionNoProxy, self).setUp()
305
    self.proxy_mode = False
306
307
308
309
310
311
312
313
314
315
316
317
318
319



class TestLocalExecution(TestExecution):

  def create_executor(self, prefix, configuration, tmp_prefix, dataformat_cache,
                      database_cache, algorithm_cache):
    return LocalExecutor(prefix, configuration, tmp_prefix,
                         dataformat_cache, database_cache, algorithm_cache)



# Don't run tests from the base class
del TestExecution