test_data.py 16.2 KB
Newer Older
1 2 3
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :

4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
###################################################################################
#                                                                                 #
# Copyright (c) 2019 Idiap Research Institute, http://www.idiap.ch/               #
# Contact: beat.support@idiap.ch                                                  #
#                                                                                 #
# Redistribution and use in source and binary forms, with or without              #
# modification, are permitted provided that the following conditions are met:     #
#                                                                                 #
# 1. Redistributions of source code must retain the above copyright notice, this  #
# list of conditions and the following disclaimer.                                #
#                                                                                 #
# 2. Redistributions in binary form must reproduce the above copyright notice,    #
# this list of conditions and the following disclaimer in the documentation       #
# and/or other materials provided with the distribution.                          #
#                                                                                 #
# 3. Neither the name of the copyright holder nor the names of its contributors   #
# may be used to endorse or promote products derived from this software without   #
# specific prior written permission.                                              #
#                                                                                 #
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND #
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED   #
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE          #
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE    #
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL      #
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR      #
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER      #
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,   #
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE   #
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.            #
#                                                                                 #
###################################################################################
35 36 37 38 39 40


import unittest
import os
import glob
import tempfile
41
import shutil
42 43 44

from ..data import mixDataIndices
from ..data import CachedDataSource
45
from ..data import CachedDataSink
46 47 48
from ..data import getAllFilenames
from ..data import foundSplitRanges
from ..dataformat import DataFormat
49
from ..database import Database
50 51 52 53

from . import prefix


54
# ----------------------------------------------------------
55 56 57 58


class TestMixDataIndices(unittest.TestCase):
    def test_one_list(self):
59
        list_of_data_indices = [[(0, 2), (3, 4), (5, 6)]]
60 61 62 63 64 65

        result = mixDataIndices(list_of_data_indices)

        self.assertEqual([(0, 2), (3, 4), (5, 6)], result)

    def test_two_identical_lists(self):
66
        list_of_data_indices = [[(0, 2), (3, 4), (5, 6)], [(0, 2), (3, 4), (5, 6)]]
67 68 69 70 71 72

        result = mixDataIndices(list_of_data_indices)

        self.assertEqual([(0, 2), (3, 4), (5, 6)], result)

    def test_two_synchronized_lists(self):
73
        list_of_data_indices = [[(0, 2), (3, 4), (5, 6)], [(0, 4), (5, 6)]]
74 75 76 77 78 79

        result = mixDataIndices(list_of_data_indices)

        self.assertEqual([(0, 2), (3, 4), (5, 6)], result)

    def test_two_desynchronized_lists(self):
80
        list_of_data_indices = [[(0, 2), (3, 4), (5, 6)], [(0, 1), (2, 4), (5, 6)]]
81 82 83 84 85 86

        result = mixDataIndices(list_of_data_indices)

        self.assertEqual([(0, 1), (2, 2), (3, 4), (5, 6)], result)


87
# ----------------------------------------------------------
88 89 90 91


class TestCachedDataBase(unittest.TestCase):
    def setUp(self):
92 93
        testfile = tempfile.NamedTemporaryFile(prefix=__name__, suffix=".data")
        testfile.close()  # preserve only the name
94 95 96 97 98
        self.filename = testfile.name

    def tearDown(self):
        basename, ext = os.path.splitext(self.filename)
        filenames = [self.filename]
99
        filenames += glob.glob(basename + "*")
100 101 102 103 104 105 106 107 108
        for filename in filenames:
            if os.path.exists(filename):
                os.unlink(filename)

    def writeData(self, dataformat_name, start_index=0, end_index=10):
        dataformat = DataFormat(prefix, dataformat_name)
        self.assertTrue(dataformat.valid)

        data_sink = CachedDataSink()
109 110 111
        self.assertTrue(
            data_sink.setup(self.filename, dataformat, start_index, end_index)
        )
112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128

        all_data = []
        for i in range(start_index, end_index + 1):
            data = dataformat.type()
            data_sink.write(data, i, i)
            all_data.append(data)

        (nb_bytes, duration) = data_sink.statistics()
        self.assertTrue(nb_bytes > 0)
        self.assertTrue(duration > 0)

        data_sink.close()
        del data_sink

        return all_data


129
# ----------------------------------------------------------
130 131 132 133


class TestGetAllFilenames(TestCachedDataBase):
    def test_one_complete_data_file(self):
134
        self.writeData("user/single_integer/1", 0, 9)
135

136 137 138 139 140 141
        (
            data_filenames,
            indices_filenames,
            data_checksum_filenames,
            indices_checksum_filenames,
        ) = getAllFilenames(self.filename)
142 143 144 145 146 147 148

        self.assertEqual(1, len(data_filenames))
        self.assertEqual(1, len(indices_filenames))
        self.assertEqual(1, len(data_checksum_filenames))
        self.assertEqual(1, len(indices_checksum_filenames))

    def test_three_complete_data_files(self):
149 150 151
        self.writeData("user/single_integer/1", 0, 9)
        self.writeData("user/single_integer/1", 10, 19)
        self.writeData("user/single_integer/1", 20, 29)
152

153 154 155 156 157 158
        (
            data_filenames,
            indices_filenames,
            data_checksum_filenames,
            indices_checksum_filenames,
        ) = getAllFilenames(self.filename)
159 160 161 162 163 164 165

        self.assertEqual(3, len(data_filenames))
        self.assertEqual(3, len(indices_filenames))
        self.assertEqual(3, len(data_checksum_filenames))
        self.assertEqual(3, len(indices_checksum_filenames))

    def test_one_partial_data_file(self):
166
        self.writeData("user/single_integer/1", 0, 9)
167

168 169 170 171 172 173
        (
            data_filenames,
            indices_filenames,
            data_checksum_filenames,
            indices_checksum_filenames,
        ) = getAllFilenames(self.filename, 2, 6)
174 175 176 177 178 179 180

        self.assertEqual(1, len(data_filenames))
        self.assertEqual(1, len(indices_filenames))
        self.assertEqual(1, len(data_checksum_filenames))
        self.assertEqual(1, len(indices_checksum_filenames))

    def test_three_partial_data_files_1(self):
181 182 183
        self.writeData("user/single_integer/1", 0, 9)
        self.writeData("user/single_integer/1", 10, 19)
        self.writeData("user/single_integer/1", 20, 29)
184

185 186 187 188 189 190
        (
            data_filenames,
            indices_filenames,
            data_checksum_filenames,
            indices_checksum_filenames,
        ) = getAllFilenames(self.filename, 14, 18)
191 192 193 194 195 196 197

        self.assertEqual(1, len(data_filenames))
        self.assertEqual(1, len(indices_filenames))
        self.assertEqual(1, len(data_checksum_filenames))
        self.assertEqual(1, len(indices_checksum_filenames))

    def test_three_partial_data_files_2(self):
198 199 200
        self.writeData("user/single_integer/1", 0, 9)
        self.writeData("user/single_integer/1", 10, 19)
        self.writeData("user/single_integer/1", 20, 29)
201

202 203 204 205 206 207
        (
            data_filenames,
            indices_filenames,
            data_checksum_filenames,
            indices_checksum_filenames,
        ) = getAllFilenames(self.filename, 4, 18)
208 209 210 211 212 213 214

        self.assertEqual(2, len(data_filenames))
        self.assertEqual(2, len(indices_filenames))
        self.assertEqual(2, len(data_checksum_filenames))
        self.assertEqual(2, len(indices_checksum_filenames))

    def test_three_partial_data_files_3(self):
215 216 217
        self.writeData("user/single_integer/1", 0, 9)
        self.writeData("user/single_integer/1", 10, 19)
        self.writeData("user/single_integer/1", 20, 29)
218

219 220 221 222 223 224
        (
            data_filenames,
            indices_filenames,
            data_checksum_filenames,
            indices_checksum_filenames,
        ) = getAllFilenames(self.filename, 4, 28)
225 226 227 228 229 230 231

        self.assertEqual(3, len(data_filenames))
        self.assertEqual(3, len(indices_filenames))
        self.assertEqual(3, len(data_checksum_filenames))
        self.assertEqual(3, len(indices_checksum_filenames))


232
# ----------------------------------------------------------
233 234


235
class TestCachedDataSource(TestCachedDataBase):
236 237 238 239 240 241 242 243 244
    def check_valid_indices(self, cached_file):
        for i in range(0, len(cached_file)):
            (data, start_index, end_index) = cached_file[i]
            self.assertTrue(data is not None)
            self.assertEqual(i + cached_file.first_data_index(), start_index)
            self.assertEqual(i + cached_file.first_data_index(), end_index)

    def check_valid_data_indices(self, cached_file):
        for i in range(0, len(cached_file)):
245 246 247
            (data, start_index, end_index) = cached_file.getAtDataIndex(
                i + cached_file.first_data_index()
            )
248 249 250 251 252 253 254 255 256 257 258 259 260
            self.assertTrue(data is not None)
            self.assertEqual(i + cached_file.first_data_index(), start_index)
            self.assertEqual(i + cached_file.first_data_index(), end_index)

    def check_invalid_indices(self, cached_file):
        # Invalid indices
        (data, start_index, end_index) = cached_file[-1]
        self.assertTrue(data is None)

        (data, start_index, end_index) = cached_file[len(cached_file)]
        self.assertTrue(data is None)

        # Invalid data indices
261 262 263
        (data, start_index, end_index) = cached_file.getAtDataIndex(
            cached_file.first_data_index() - 1
        )
264 265
        self.assertTrue(data is None)

266 267 268
        (data, start_index, end_index) = cached_file.getAtDataIndex(
            cached_file.last_data_index() + 1
        )
269 270 271
        self.assertTrue(data is None)

    def test_one_complete_data_file(self):
272
        self.writeData("user/single_integer/1", 0, 9)
273

274
        cached_file = CachedDataSource()
275 276 277 278 279 280 281 282 283
        cached_file.setup(self.filename, prefix)

        self.assertEqual(10, len(cached_file))

        self.check_valid_indices(cached_file)
        self.check_valid_data_indices(cached_file)
        self.check_invalid_indices(cached_file)

    def test_three_complete_data_files(self):
284 285 286
        self.writeData("user/single_integer/1", 0, 9)
        self.writeData("user/single_integer/1", 10, 19)
        self.writeData("user/single_integer/1", 20, 29)
287

288
        cached_file = CachedDataSource()
289 290 291 292 293 294 295 296 297
        cached_file.setup(self.filename, prefix)

        self.assertEqual(30, len(cached_file))

        self.check_valid_indices(cached_file)
        self.check_valid_data_indices(cached_file)
        self.check_invalid_indices(cached_file)

    def test_one_partial_data_file(self):
298
        self.writeData("user/single_integer/1", 0, 9)
299

300
        cached_file = CachedDataSource()
301 302 303 304 305 306 307 308 309
        cached_file.setup(self.filename, prefix, 2, 6)

        self.assertEqual(5, len(cached_file))

        self.check_valid_indices(cached_file)
        self.check_valid_data_indices(cached_file)
        self.check_invalid_indices(cached_file)

    def test_three_partial_data_files_1(self):
310 311 312
        self.writeData("user/single_integer/1", 0, 9)
        self.writeData("user/single_integer/1", 10, 19)
        self.writeData("user/single_integer/1", 20, 29)
313

314
        cached_file = CachedDataSource()
315 316 317 318 319 320 321 322 323
        cached_file.setup(self.filename, prefix, 14, 18)

        self.assertEqual(5, len(cached_file))

        self.check_valid_indices(cached_file)
        self.check_valid_data_indices(cached_file)
        self.check_invalid_indices(cached_file)

    def test_three_partial_data_files_2(self):
324 325 326
        self.writeData("user/single_integer/1", 0, 9)
        self.writeData("user/single_integer/1", 10, 19)
        self.writeData("user/single_integer/1", 20, 29)
327

328
        cached_file = CachedDataSource()
329 330 331 332 333 334 335 336 337
        cached_file.setup(self.filename, prefix, 4, 18)

        self.assertEqual(15, len(cached_file))

        self.check_valid_indices(cached_file)
        self.check_valid_data_indices(cached_file)
        self.check_invalid_indices(cached_file)

    def test_three_partial_data_files_3(self):
338 339 340
        self.writeData("user/single_integer/1", 0, 9)
        self.writeData("user/single_integer/1", 10, 19)
        self.writeData("user/single_integer/1", 20, 29)
341

342
        cached_file = CachedDataSource()
343 344 345 346 347 348 349 350 351
        cached_file.setup(self.filename, prefix, 4, 28)

        self.assertEqual(25, len(cached_file))

        self.check_valid_indices(cached_file)
        self.check_valid_data_indices(cached_file)
        self.check_invalid_indices(cached_file)


352
# ----------------------------------------------------------
353 354


355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370
class TestDatabaseOutputDataSource(unittest.TestCase):
    def setUp(self):
        self.cache_root = tempfile.mkdtemp(prefix=__name__)

    def tearDown(self):
        shutil.rmtree(self.cache_root)

    def check_valid_indices(self, data_source):
        for i in range(0, len(data_source)):
            (data, start_index, end_index) = data_source[i]
            self.assertTrue(data is not None)
            self.assertEqual(i + data_source.first_data_index(), start_index)
            self.assertEqual(i + data_source.first_data_index(), end_index)

    def check_valid_data_indices(self, data_source):
        for i in range(0, len(data_source)):
371 372 373
            (data, start_index, end_index) = data_source.getAtDataIndex(
                i + data_source.first_data_index()
            )
374 375 376 377 378 379 380 381 382 383 384 385 386
            self.assertTrue(data is not None)
            self.assertEqual(i + data_source.first_data_index(), start_index)
            self.assertEqual(i + data_source.first_data_index(), end_index)

    def check_invalid_indices(self, data_source):
        # Invalid indices
        (data, start_index, end_index) = data_source[-1]
        self.assertTrue(data is None)

        (data, start_index, end_index) = data_source[len(data_source)]
        self.assertTrue(data is None)

        # Invalid data indices
387 388 389
        (data, start_index, end_index) = data_source.getAtDataIndex(
            data_source.first_data_index() - 1
        )
390 391
        self.assertTrue(data is None)

392 393 394
        (data, start_index, end_index) = data_source.getAtDataIndex(
            data_source.last_data_index() + 1
        )
395 396 397
        self.assertTrue(data is None)

    def test(self):
398
        db = Database(prefix, "integers_db/1")
399 400
        self.assertTrue(db.valid)

401 402 403
        view = db.view("double", "double")
        view.index(os.path.join(self.cache_root, "data.db"))
        view.setup(os.path.join(self.cache_root, "data.db"), pack=False)
404 405

        self.assertTrue(view.data_sources is not None)
406
        self.assertEqual(len(view.data_sources), 3)
407 408 409 410 411 412 413 414 415

        for output_name, data_source in view.data_sources.items():
            self.assertEqual(9, len(data_source))

            self.check_valid_indices(data_source)
            self.check_valid_data_indices(data_source)
            self.check_invalid_indices(data_source)


416
# ----------------------------------------------------------
417 418


419 420
class TestDataSink(TestCachedDataBase):
    def test_creation(self):
421
        dataformat = DataFormat(prefix, "user/single_integer/1")
422 423 424
        self.assertTrue(dataformat.valid)

        data_sink = CachedDataSink()
425
        self.assertTrue(data_sink.setup(self.filename, dataformat, 0, 10))
426 427


428
# ----------------------------------------------------------
429 430 431 432


class TestFoundSplitRanges(unittest.TestCase):
    def test_2_splits(self):
433
        splits = [[0, 2, 4, 6, 8, 10, 12], [0, 3, 6, 9, 12]]
434 435
        n_split = 2
        ref = [(0, 5), (6, 11)]
436
        res = foundSplitRanges(splits, n_split)
437 438 439
        self.assertEqual(res, ref)

    def test_5_splits(self):
440
        splits = [[0, 2, 4, 6, 8, 10, 12, 15], [0, 3, 6, 9, 12, 15]]
441 442
        n_split = 5
        ref = [(0, 5), (6, 11), (12, 14)]
443
        res = foundSplitRanges(splits, n_split)
444
        self.assertEqual(res, ref)