test_execution.py 7.92 KB
Newer Older
André Anjos's avatar
André Anjos committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

###############################################################################
#                                                                             #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
# Contact: beat.support@idiap.ch                                              #
#                                                                             #
# This file is part of the beat.core module of the BEAT platform.             #
#                                                                             #
# Commercial License Usage                                                    #
# Licensees holding valid commercial BEAT licenses may use this file in       #
# accordance with the terms contained in a written agreement between you      #
# and Idiap. For further information contact tto@idiap.ch                     #
#                                                                             #
# Alternatively, this file may be used under the terms of the GNU Affero      #
# Public License version 3 as published by the Free Software and appearing    #
# in the file LICENSE.AGPL included in the packaging of this file.            #
# The BEAT platform is distributed in the hope that it will be useful, but    #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
#                                                                             #
# You should have received a copy of the GNU Affero Public License along      #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
#                                                                             #
###############################################################################


# Tests for experiment execution

import os
import glob
import logging
logger = logging.getLogger(__name__)

# in case you want to see the printouts dynamically, set to ``True``
if False:
  logger = logging.getLogger() #root logger
  logger.setLevel(logging.DEBUG)
  ch = logging.StreamHandler()
  ch.setLevel(logging.DEBUG)
  ch.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))
  logger.addHandler(ch)

import numpy
import nose.tools

from ..experiment import Experiment
from ..execution import Executor
from ..hash import hashFileContents
from ..data import CachedDataSource
52
from .. import async
André Anjos's avatar
André Anjos committed
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

from . import prefix, tmp_prefix
from .utils import slow, cleanup


def check_output(prefix, path):
  '''Checks if a given output exists, together with its indexes and checksums
  '''

  finalpath = os.path.join(prefix, path)
  datafiles = glob.glob(finalpath + '*.data')
  datachksums = glob.glob(finalpath + '*.data.checksum')
  indexfiles = glob.glob(finalpath + '*.index')
  indexchksums = glob.glob(finalpath + '*.index.checksum')
  assert datafiles
  nose.tools.eq_(len(datafiles), len(indexfiles))
  for k in datafiles + indexfiles:
    checksum_file = k + '.checksum'
    assert checksum_file in datachksums + indexchksums
    stored_checksum = None
    with open(checksum_file, 'rt') as f: stored_checksum = f.read().strip()
    current_checksum = hashFileContents(k)
    nose.tools.eq_(current_checksum, stored_checksum)


def load_result(executor):
  '''Loads the result of an experiment, in a single go'''

  f = CachedDataSource()
  assert f.setup(os.path.join(executor.cache,
      executor.data['result']['path'] + '.data'), executor.prefix)
  data, start, end = f.next()
  nose.tools.eq_(start, 0)
  nose.tools.eq_(start, end)
  f.close()
  return data


91
def execute(label, expected_result):
André Anjos's avatar
André Anjos committed
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
  """Executes the full experiment, block after block, returning results. If an
  error occurs, returns information about the err'd block. Otherwise, returns
  ``None``.

  This bit of code mimics the scheduler, but does everything on the local
  machine. It borrows some code from the package ``beat.cmdline``.
  """

  dataformat_cache = {}
  database_cache = {}
  algorithm_cache = {}

  experiment = Experiment(prefix, label,
          dataformat_cache, database_cache, algorithm_cache)

  assert experiment.valid, '\n  * %s' % '\n  * '.join(experiment.errors)

  scheduled = experiment.setup()

  # can we execute it?
  results = []
  for key, value in scheduled.items():
    executor = Executor(prefix, value['configuration'], tmp_prefix,
          dataformat_cache, database_cache, algorithm_cache)
    assert executor.valid, '\n  * %s' % '\n  * '.join(executor.errors)

118
119
    with executor, async.Host(use_machine='default') as host:
      result = executor.process(host)
André Anjos's avatar
André Anjos committed
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
      assert result
      assert 'status' in result
      assert 'stdout' in result
      assert 'stderr' in result
      assert 'statistics' in result
      assert 'timed_out' in result
      assert 'system_error' in result
      assert 'user_error' in result
      if result['status'] != 0:
        logger.warn("(eventual) system errors: %s", result['system_error'])
        logger.warn("(eventual) user errors: %s", result['user_error'])
        logger.warn("stdout: %s", result['stdout'])
        logger.warn("stderr: %s", result['stderr'])
        return result
      if result['system_error']:
        logger.warn("system errors: %s", result['system_error'])
        return result
      assert result['status'] == 0
      assert isinstance(result['statistics'], dict)
      assert result['statistics']

    if executor.analysis:
      check_output(tmp_prefix, executor.data['result']['path'])
      results.append(load_result(executor))
    else:
      for name, details in executor.data['outputs'].items():
        check_output(tmp_prefix, details['path'])

  # compares all results
  assert results
  for k, result in enumerate(results):
    expected = result.__class__()
    expected.from_dict(expected_result[k], casting='unsafe') #defaults=False
    assert result.isclose(expected), "%r is not close enough to %r" % (result.as_dict(), expected.as_dict())


@slow
@nose.tools.with_setup(teardown=cleanup)
def test_integers_addition_1():
  assert execute('user/user/integers_addition/1/integers_addition',
          [{'sum': 12272, 'nb': 10}]) is None

@slow
@nose.tools.with_setup(teardown=cleanup)
def test_integers_addition_2():
  assert execute('user/user/integers_addition/2/integers_addition',
          [{'sum': 18392, 'nb': 10}]) is None

@slow
@nose.tools.with_setup(teardown=cleanup)
def test_single_1_single():
  assert execute('user/user/single/1/single', [{'out_data': 42}]) is None

@slow
@nose.tools.with_setup(teardown=cleanup)
175
176
def test_single_1_add():
  assert execute('user/user/single/1/single_add', [{'out_data': 43}]) is None
André Anjos's avatar
André Anjos committed
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220

@slow
@nose.tools.with_setup(teardown=cleanup)
def test_single_1_add2():
  assert execute('user/user/single/1/single_add2', [{'out_data': 44}]) is None

@slow
@nose.tools.with_setup(teardown=cleanup)
def test_single_1_error():
  result = execute('user/user/single/1/single_error', [None])
  assert result
  nose.tools.eq_(result['status'], 1)
  assert result['user_error']
  assert 'NameError' in result['user_error']
  nose.tools.eq_(result['system_error'], '')

@slow
@nose.tools.with_setup(teardown=cleanup)
def test_single_1_large():
  assert execute('user/user/single/1/single_large', [{'out_data': 2.0}]) is None

@slow
@nose.tools.with_setup(teardown=cleanup)
def test_double_1():
  assert execute('user/user/double/1/double', [{'out_data': 42}]) is None

@slow
@nose.tools.with_setup(teardown=cleanup)
def test_triangle_1():
  assert execute('user/user/triangle/1/triangle', [{'out_data': 42}]) is None

@slow
@nose.tools.with_setup(teardown=cleanup)
def test_too_many_nexts():
  result = execute('user/user/triangle/1/too_many_nexts', [None])
  assert result
  nose.tools.eq_(result['status'], 9)
  assert result['user_error']
  assert 'no more data' in result['user_error']

@slow
@nose.tools.with_setup(teardown=cleanup)
def test_double_triangle_1():
  assert execute('user/user/double_triangle/1/double_triangle', [{'out_data': 42}]) is None