Skip to content

beat.backend.python
beat.backend.python
Commit c4c9edca authored by Samuel GAIST's avatar Samuel GAIST
Browse files
Options

[test][data] Pre-commit cleanup

parent 45dccc3f
Hide whitespace changes
Inline Side-by-side
Showing with 108 additions and 113 deletions
beat/backend/python/test/test_data.py
View file @ c4c9edca
...@@ -45,87 +45,70 @@ from ..data import CachedDataSource ...@@ -45,87 +45,70 @@ from ..data import CachedDataSource
from ..data import CachedDataSink from ..data import CachedDataSink
from ..data import getAllFilenames from ..data import getAllFilenames
from ..data import foundSplitRanges from ..data import foundSplitRanges
from ..hash import hashFileContents
from ..dataformat import DataFormat from ..dataformat import DataFormat
from ..database import Database from ..database import Database
from . import prefix from . import prefix
#---------------------------------------------------------- # ----------------------------------------------------------
class TestMixDataIndices(unittest.TestCase): class TestMixDataIndices(unittest.TestCase):
def test_one_list(self): def test_one_list(self):
list_of_data_indices = [ list_of_data_indices = [[(0, 2), (3, 4), (5, 6)]]
[(0, 2), (3, 4), (5, 6)]
]
result = mixDataIndices(list_of_data_indices) result = mixDataIndices(list_of_data_indices)
self.assertEqual([(0, 2), (3, 4), (5, 6)], result) self.assertEqual([(0, 2), (3, 4), (5, 6)], result)
def test_two_identical_lists(self): def test_two_identical_lists(self):
list_of_data_indices = [ list_of_data_indices = [[(0, 2), (3, 4), (5, 6)], [(0, 2), (3, 4), (5, 6)]]
[(0, 2), (3, 4), (5, 6)],
[(0, 2), (3, 4), (5, 6)],
]
result = mixDataIndices(list_of_data_indices) result = mixDataIndices(list_of_data_indices)
self.assertEqual([(0, 2), (3, 4), (5, 6)], result) self.assertEqual([(0, 2), (3, 4), (5, 6)], result)
def test_two_synchronized_lists(self): def test_two_synchronized_lists(self):
list_of_data_indices = [ list_of_data_indices = [[(0, 2), (3, 4), (5, 6)], [(0, 4), (5, 6)]]
[(0, 2), (3, 4), (5, 6)],
[(0, 4), (5, 6)],
]
result = mixDataIndices(list_of_data_indices) result = mixDataIndices(list_of_data_indices)
self.assertEqual([(0, 2), (3, 4), (5, 6)], result) self.assertEqual([(0, 2), (3, 4), (5, 6)], result)
def test_two_desynchronized_lists(self): def test_two_desynchronized_lists(self):
list_of_data_indices = [ list_of_data_indices = [[(0, 2), (3, 4), (5, 6)], [(0, 1), (2, 4), (5, 6)]]
[(0, 2), (3, 4), (5, 6)],
[(0, 1), (2, 4), (5, 6)],
]
result = mixDataIndices(list_of_data_indices) result = mixDataIndices(list_of_data_indices)
self.assertEqual([(0, 1), (2, 2), (3, 4), (5, 6)], result) self.assertEqual([(0, 1), (2, 2), (3, 4), (5, 6)], result)
#---------------------------------------------------------- # ----------------------------------------------------------
class TestCachedDataBase(unittest.TestCase): class TestCachedDataBase(unittest.TestCase):
def setUp(self): def setUp(self):
testfile = tempfile.NamedTemporaryFile(prefix=__name__, suffix='.data') testfile = tempfile.NamedTemporaryFile(prefix=__name__, suffix=".data")
testfile.close() # preserve only the name testfile.close() # preserve only the name
self.filename = testfile.name self.filename = testfile.name
def tearDown(self): def tearDown(self):
basename, ext = os.path.splitext(self.filename) basename, ext = os.path.splitext(self.filename)
filenames = [self.filename] filenames = [self.filename]
filenames += glob.glob(basename + '*') filenames += glob.glob(basename + "*")
for filename in filenames: for filename in filenames:
if os.path.exists(filename): if os.path.exists(filename):
os.unlink(filename) os.unlink(filename)
def writeData(self, dataformat_name, start_index=0, end_index=10): def writeData(self, dataformat_name, start_index=0, end_index=10):
dataformat = DataFormat(prefix, dataformat_name) dataformat = DataFormat(prefix, dataformat_name)
self.assertTrue(dataformat.valid) self.assertTrue(dataformat.valid)
data_sink = CachedDataSink() data_sink = CachedDataSink()
self.assertTrue(data_sink.setup(self.filename, dataformat, start_index, end_index)) self.assertTrue(
data_sink.setup(self.filename, dataformat, start_index, end_index)
)
all_data = [] all_data = []
for i in range(start_index, end_index + 1): for i in range(start_index, end_index + 1):
...@@ -143,84 +126,102 @@ class TestCachedDataBase(unittest.TestCase): ...@@ -143,84 +126,102 @@ class TestCachedDataBase(unittest.TestCase):
return all_data return all_data
#---------------------------------------------------------- # ----------------------------------------------------------
class TestGetAllFilenames(TestCachedDataBase): class TestGetAllFilenames(TestCachedDataBase):
def test_one_complete_data_file(self): def test_one_complete_data_file(self):
self.writeData('user/single_integer/1', 0, 9) self.writeData("user/single_integer/1", 0, 9)
(data_filenames, indices_filenames, data_checksum_filenames, indices_checksum_filenames) = \ (
getAllFilenames(self.filename) data_filenames,
indices_filenames,
data_checksum_filenames,
indices_checksum_filenames,
) = getAllFilenames(self.filename)
self.assertEqual(1, len(data_filenames)) self.assertEqual(1, len(data_filenames))
self.assertEqual(1, len(indices_filenames)) self.assertEqual(1, len(indices_filenames))
self.assertEqual(1, len(data_checksum_filenames)) self.assertEqual(1, len(data_checksum_filenames))
self.assertEqual(1, len(indices_checksum_filenames)) self.assertEqual(1, len(indices_checksum_filenames))
def test_three_complete_data_files(self): def test_three_complete_data_files(self):
self.writeData('user/single_integer/1', 0, 9) self.writeData("user/single_integer/1", 0, 9)
self.writeData('user/single_integer/1', 10, 19) self.writeData("user/single_integer/1", 10, 19)
self.writeData('user/single_integer/1', 20, 29) self.writeData("user/single_integer/1", 20, 29)
(data_filenames, indices_filenames, data_checksum_filenames, indices_checksum_filenames) = \ (
getAllFilenames(self.filename) data_filenames,
indices_filenames,
data_checksum_filenames,
indices_checksum_filenames,
) = getAllFilenames(self.filename)
self.assertEqual(3, len(data_filenames)) self.assertEqual(3, len(data_filenames))
self.assertEqual(3, len(indices_filenames)) self.assertEqual(3, len(indices_filenames))
self.assertEqual(3, len(data_checksum_filenames)) self.assertEqual(3, len(data_checksum_filenames))
self.assertEqual(3, len(indices_checksum_filenames)) self.assertEqual(3, len(indices_checksum_filenames))
def test_one_partial_data_file(self): def test_one_partial_data_file(self):
self.writeData('user/single_integer/1', 0, 9) self.writeData("user/single_integer/1", 0, 9)
(data_filenames, indices_filenames, data_checksum_filenames, indices_checksum_filenames) = \ (
getAllFilenames(self.filename, 2, 6) data_filenames,
indices_filenames,
data_checksum_filenames,
indices_checksum_filenames,
) = getAllFilenames(self.filename, 2, 6)
self.assertEqual(1, len(data_filenames)) self.assertEqual(1, len(data_filenames))
self.assertEqual(1, len(indices_filenames)) self.assertEqual(1, len(indices_filenames))
self.assertEqual(1, len(data_checksum_filenames)) self.assertEqual(1, len(data_checksum_filenames))
self.assertEqual(1, len(indices_checksum_filenames)) self.assertEqual(1, len(indices_checksum_filenames))
def test_three_partial_data_files_1(self): def test_three_partial_data_files_1(self):
self.writeData('user/single_integer/1', 0, 9) self.writeData("user/single_integer/1", 0, 9)
self.writeData('user/single_integer/1', 10, 19) self.writeData("user/single_integer/1", 10, 19)
self.writeData('user/single_integer/1', 20, 29) self.writeData("user/single_integer/1", 20, 29)
(data_filenames, indices_filenames, data_checksum_filenames, indices_checksum_filenames) = \ (
getAllFilenames(self.filename, 14, 18) data_filenames,
indices_filenames,
data_checksum_filenames,
indices_checksum_filenames,
) = getAllFilenames(self.filename, 14, 18)
self.assertEqual(1, len(data_filenames)) self.assertEqual(1, len(data_filenames))
self.assertEqual(1, len(indices_filenames)) self.assertEqual(1, len(indices_filenames))
self.assertEqual(1, len(data_checksum_filenames)) self.assertEqual(1, len(data_checksum_filenames))
self.assertEqual(1, len(indices_checksum_filenames)) self.assertEqual(1, len(indices_checksum_filenames))
def test_three_partial_data_files_2(self): def test_three_partial_data_files_2(self):
self.writeData('user/single_integer/1', 0, 9) self.writeData("user/single_integer/1", 0, 9)
self.writeData('user/single_integer/1', 10, 19) self.writeData("user/single_integer/1", 10, 19)
self.writeData('user/single_integer/1', 20, 29) self.writeData("user/single_integer/1", 20, 29)
(data_filenames, indices_filenames, data_checksum_filenames, indices_checksum_filenames) = \ (
getAllFilenames(self.filename, 4, 18) data_filenames,
indices_filenames,
data_checksum_filenames,
indices_checksum_filenames,
) = getAllFilenames(self.filename, 4, 18)
self.assertEqual(2, len(data_filenames)) self.assertEqual(2, len(data_filenames))
self.assertEqual(2, len(indices_filenames)) self.assertEqual(2, len(indices_filenames))
self.assertEqual(2, len(data_checksum_filenames)) self.assertEqual(2, len(data_checksum_filenames))
self.assertEqual(2, len(indices_checksum_filenames)) self.assertEqual(2, len(indices_checksum_filenames))
def test_three_partial_data_files_3(self): def test_three_partial_data_files_3(self):
self.writeData('user/single_integer/1', 0, 9) self.writeData("user/single_integer/1", 0, 9)
self.writeData('user/single_integer/1', 10, 19) self.writeData("user/single_integer/1", 10, 19)
self.writeData('user/single_integer/1', 20, 29) self.writeData("user/single_integer/1", 20, 29)
(data_filenames, indices_filenames, data_checksum_filenames, indices_checksum_filenames) = \ (
getAllFilenames(self.filename, 4, 28) data_filenames,
indices_filenames,
data_checksum_filenames,
indices_checksum_filenames,
) = getAllFilenames(self.filename, 4, 28)
self.assertEqual(3, len(data_filenames)) self.assertEqual(3, len(data_filenames))
self.assertEqual(3, len(indices_filenames)) self.assertEqual(3, len(indices_filenames))
...@@ -228,11 +229,10 @@ class TestGetAllFilenames(TestCachedDataBase): ...@@ -228,11 +229,10 @@ class TestGetAllFilenames(TestCachedDataBase):
self.assertEqual(3, len(indices_checksum_filenames)) self.assertEqual(3, len(indices_checksum_filenames))
#---------------------------------------------------------- # ----------------------------------------------------------
class TestCachedDataSource(TestCachedDataBase): class TestCachedDataSource(TestCachedDataBase):
def check_valid_indices(self, cached_file): def check_valid_indices(self, cached_file):
for i in range(0, len(cached_file)): for i in range(0, len(cached_file)):
(data, start_index, end_index) = cached_file[i] (data, start_index, end_index) = cached_file[i]
...@@ -240,15 +240,15 @@ class TestCachedDataSource(TestCachedDataBase): ...@@ -240,15 +240,15 @@ class TestCachedDataSource(TestCachedDataBase):
self.assertEqual(i + cached_file.first_data_index(), start_index) self.assertEqual(i + cached_file.first_data_index(), start_index)
self.assertEqual(i + cached_file.first_data_index(), end_index) self.assertEqual(i + cached_file.first_data_index(), end_index)
def check_valid_data_indices(self, cached_file): def check_valid_data_indices(self, cached_file):
for i in range(0, len(cached_file)): for i in range(0, len(cached_file)):
(data, start_index, end_index) = cached_file.getAtDataIndex(i + cached_file.first_data_index()) (data, start_index, end_index) = cached_file.getAtDataIndex(
i + cached_file.first_data_index()
)
self.assertTrue(data is not None) self.assertTrue(data is not None)
self.assertEqual(i + cached_file.first_data_index(), start_index) self.assertEqual(i + cached_file.first_data_index(), start_index)
self.assertEqual(i + cached_file.first_data_index(), end_index) self.assertEqual(i + cached_file.first_data_index(), end_index)
def check_invalid_indices(self, cached_file): def check_invalid_indices(self, cached_file):
# Invalid indices # Invalid indices
(data, start_index, end_index) = cached_file[-1] (data, start_index, end_index) = cached_file[-1]
...@@ -258,15 +258,18 @@ class TestCachedDataSource(TestCachedDataBase): ...@@ -258,15 +258,18 @@ class TestCachedDataSource(TestCachedDataBase):
self.assertTrue(data is None) self.assertTrue(data is None)
# Invalid data indices # Invalid data indices
(data, start_index, end_index) = cached_file.getAtDataIndex(cached_file.first_data_index() - 1) (data, start_index, end_index) = cached_file.getAtDataIndex(
cached_file.first_data_index() - 1
)
self.assertTrue(data is None) self.assertTrue(data is None)
(data, start_index, end_index) = cached_file.getAtDataIndex(cached_file.last_data_index() + 1) (data, start_index, end_index) = cached_file.getAtDataIndex(
cached_file.last_data_index() + 1
)
self.assertTrue(data is None) self.assertTrue(data is None)
def test_one_complete_data_file(self): def test_one_complete_data_file(self):
self.writeData('user/single_integer/1', 0, 9) self.writeData("user/single_integer/1", 0, 9)
cached_file = CachedDataSource() cached_file = CachedDataSource()
cached_file.setup(self.filename, prefix) cached_file.setup(self.filename, prefix)
...@@ -277,11 +280,10 @@ class TestCachedDataSource(TestCachedDataBase): ...@@ -277,11 +280,10 @@ class TestCachedDataSource(TestCachedDataBase):
self.check_valid_data_indices(cached_file) self.check_valid_data_indices(cached_file)
self.check_invalid_indices(cached_file) self.check_invalid_indices(cached_file)
def test_three_complete_data_files(self): def test_three_complete_data_files(self):
self.writeData('user/single_integer/1', 0, 9) self.writeData("user/single_integer/1", 0, 9)
self.writeData('user/single_integer/1', 10, 19) self.writeData("user/single_integer/1", 10, 19)
self.writeData('user/single_integer/1', 20, 29) self.writeData("user/single_integer/1", 20, 29)
cached_file = CachedDataSource() cached_file = CachedDataSource()
cached_file.setup(self.filename, prefix) cached_file.setup(self.filename, prefix)
...@@ -292,9 +294,8 @@ class TestCachedDataSource(TestCachedDataBase): ...@@ -292,9 +294,8 @@ class TestCachedDataSource(TestCachedDataBase):
self.check_valid_data_indices(cached_file) self.check_valid_data_indices(cached_file)
self.check_invalid_indices(cached_file) self.check_invalid_indices(cached_file)
def test_one_partial_data_file(self): def test_one_partial_data_file(self):
self.writeData('user/single_integer/1', 0, 9) self.writeData("user/single_integer/1", 0, 9)
cached_file = CachedDataSource() cached_file = CachedDataSource()
cached_file.setup(self.filename, prefix, 2, 6) cached_file.setup(self.filename, prefix, 2, 6)
...@@ -305,11 +306,10 @@ class TestCachedDataSource(TestCachedDataBase): ...@@ -305,11 +306,10 @@ class TestCachedDataSource(TestCachedDataBase):
self.check_valid_data_indices(cached_file) self.check_valid_data_indices(cached_file)
self.check_invalid_indices(cached_file) self.check_invalid_indices(cached_file)
def test_three_partial_data_files_1(self): def test_three_partial_data_files_1(self):
self.writeData('user/single_integer/1', 0, 9) self.writeData("user/single_integer/1", 0, 9)
self.writeData('user/single_integer/1', 10, 19) self.writeData("user/single_integer/1", 10, 19)
self.writeData('user/single_integer/1', 20, 29) self.writeData("user/single_integer/1", 20, 29)
cached_file = CachedDataSource() cached_file = CachedDataSource()
cached_file.setup(self.filename, prefix, 14, 18) cached_file.setup(self.filename, prefix, 14, 18)
...@@ -320,11 +320,10 @@ class TestCachedDataSource(TestCachedDataBase): ...@@ -320,11 +320,10 @@ class TestCachedDataSource(TestCachedDataBase):
self.check_valid_data_indices(cached_file) self.check_valid_data_indices(cached_file)
self.check_invalid_indices(cached_file) self.check_invalid_indices(cached_file)
def test_three_partial_data_files_2(self): def test_three_partial_data_files_2(self):
self.writeData('user/single_integer/1', 0, 9) self.writeData("user/single_integer/1", 0, 9)
self.writeData('user/single_integer/1', 10, 19) self.writeData("user/single_integer/1", 10, 19)
self.writeData('user/single_integer/1', 20, 29) self.writeData("user/single_integer/1", 20, 29)
cached_file = CachedDataSource() cached_file = CachedDataSource()
cached_file.setup(self.filename, prefix, 4, 18) cached_file.setup(self.filename, prefix, 4, 18)
...@@ -335,11 +334,10 @@ class TestCachedDataSource(TestCachedDataBase): ...@@ -335,11 +334,10 @@ class TestCachedDataSource(TestCachedDataBase):
self.check_valid_data_indices(cached_file) self.check_valid_data_indices(cached_file)
self.check_invalid_indices(cached_file) self.check_invalid_indices(cached_file)
def test_three_partial_data_files_3(self): def test_three_partial_data_files_3(self):
self.writeData('user/single_integer/1', 0, 9) self.writeData("user/single_integer/1", 0, 9)
self.writeData('user/single_integer/1', 10, 19) self.writeData("user/single_integer/1", 10, 19)
self.writeData('user/single_integer/1', 20, 29) self.writeData("user/single_integer/1", 20, 29)
cached_file = CachedDataSource() cached_file = CachedDataSource()
cached_file.setup(self.filename, prefix, 4, 28) cached_file.setup(self.filename, prefix, 4, 28)
...@@ -351,19 +349,16 @@ class TestCachedDataSource(TestCachedDataBase): ...@@ -351,19 +349,16 @@ class TestCachedDataSource(TestCachedDataBase):
self.check_invalid_indices(cached_file) self.check_invalid_indices(cached_file)
#---------------------------------------------------------- # ----------------------------------------------------------
class TestDatabaseOutputDataSource(unittest.TestCase): class TestDatabaseOutputDataSource(unittest.TestCase):
def setUp(self): def setUp(self):
self.cache_root = tempfile.mkdtemp(prefix=__name__) self.cache_root = tempfile.mkdtemp(prefix=__name__)
def tearDown(self): def tearDown(self):
shutil.rmtree(self.cache_root) shutil.rmtree(self.cache_root)
def check_valid_indices(self, data_source): def check_valid_indices(self, data_source):
for i in range(0, len(data_source)): for i in range(0, len(data_source)):
(data, start_index, end_index) = data_source[i] (data, start_index, end_index) = data_source[i]
...@@ -371,15 +366,15 @@ class TestDatabaseOutputDataSource(unittest.TestCase): ...@@ -371,15 +366,15 @@ class TestDatabaseOutputDataSource(unittest.TestCase):
self.assertEqual(i + data_source.first_data_index(), start_index) self.assertEqual(i + data_source.first_data_index(), start_index)
self.assertEqual(i + data_source.first_data_index(), end_index) self.assertEqual(i + data_source.first_data_index(), end_index)
def check_valid_data_indices(self, data_source): def check_valid_data_indices(self, data_source):
for i in range(0, len(data_source)): for i in range(0, len(data_source)):
(data, start_index, end_index) = data_source.getAtDataIndex(i + data_source.first_data_index()) (data, start_index, end_index) = data_source.getAtDataIndex(
i + data_source.first_data_index()
)
self.assertTrue(data is not None) self.assertTrue(data is not None)
self.assertEqual(i + data_source.first_data_index(), start_index) self.assertEqual(i + data_source.first_data_index(), start_index)
self.assertEqual(i + data_source.first_data_index(), end_index) self.assertEqual(i + data_source.first_data_index(), end_index)
def check_invalid_indices(self, data_source): def check_invalid_indices(self, data_source):
# Invalid indices # Invalid indices
(data, start_index, end_index) = data_source[-1] (data, start_index, end_index) = data_source[-1]
...@@ -389,20 +384,23 @@ class TestDatabaseOutputDataSource(unittest.TestCase): ...@@ -389,20 +384,23 @@ class TestDatabaseOutputDataSource(unittest.TestCase):
self.assertTrue(data is None) self.assertTrue(data is None)
# Invalid data indices # Invalid data indices
(data, start_index, end_index) = data_source.getAtDataIndex(data_source.first_data_index() - 1) (data, start_index, end_index) = data_source.getAtDataIndex(
data_source.first_data_index() - 1
)
self.assertTrue(data is None) self.assertTrue(data is None)
(data, start_index, end_index) = data_source.getAtDataIndex(data_source.last_data_index() + 1) (data, start_index, end_index) = data_source.getAtDataIndex(
data_source.last_data_index() + 1
)
self.assertTrue(data is None) self.assertTrue(data is None)
def test(self): def test(self):
db = Database(prefix, 'integers_db/1') db = Database(prefix, "integers_db/1")
self.assertTrue(db.valid) self.assertTrue(db.valid)
view = db.view('double', 'double') view = db.view("double", "double")
view.index(os.path.join(self.cache_root, 'data.db')) view.index(os.path.join(self.cache_root, "data.db"))
view.setup(os.path.join(self.cache_root, 'data.db'), pack=False) view.setup(os.path.join(self.cache_root, "data.db"), pack=False)
self.assertTrue(view.data_sources is not None) self.assertTrue(view.data_sources is not None)
self.assertEqual(len(view.data_sources), 3) self.assertEqual(len(view.data_sources), 3)
...@@ -415,35 +413,32 @@ class TestDatabaseOutputDataSource(unittest.TestCase): ...@@ -415,35 +413,32 @@ class TestDatabaseOutputDataSource(unittest.TestCase):
self.check_invalid_indices(data_source) self.check_invalid_indices(data_source)
#---------------------------------------------------------- # ----------------------------------------------------------
class TestDataSink(TestCachedDataBase): class TestDataSink(TestCachedDataBase):
def test_creation(self): def test_creation(self):
dataformat = DataFormat(prefix, 'user/single_integer/1') dataformat = DataFormat(prefix, "user/single_integer/1")
self.assertTrue(dataformat.valid) self.assertTrue(dataformat.valid)
data_sink = CachedDataSink() data_sink = CachedDataSink()
self.assertTrue(data_sink.setup(self.filename, dataformat, 0, 10)) self.assertTrue(data_sink.setup(self.filename, dataformat, 0, 10))
#---------------------------------------------------------- # ----------------------------------------------------------
class TestFoundSplitRanges(unittest.TestCase): class TestFoundSplitRanges(unittest.TestCase):
def test_2_splits(self): def test_2_splits(self):
l = [[0,2,4,6,8,10,12], [0,3,6,9,12]] splits = [[0, 2, 4, 6, 8, 10, 12], [0, 3, 6, 9, 12]]
n_split = 2 n_split = 2
ref = [(0, 5), (6, 11)] ref = [(0, 5), (6, 11)]
res = foundSplitRanges(l, n_split) res = foundSplitRanges(splits, n_split)
self.assertEqual(res, ref) self.assertEqual(res, ref)
def test_5_splits(self): def test_5_splits(self):
l = [[0,2,4,6,8,10,12,15], [0,3,6,9,12,15]] splits = [[0, 2, 4, 6, 8, 10, 12, 15], [0, 3, 6, 9, 12, 15]]
n_split = 5 n_split = 5
ref = [(0, 5), (6, 11), (12, 14)] ref = [(0, 5), (6, 11), (12, 14)]
res = foundSplitRanges(l, n_split) res = foundSplitRanges(splits, n_split)
self.assertEqual(res, ref) self.assertEqual(res, ref)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment