Commit 39e23b21 authored by Samuel GAIST's avatar Samuel GAIST

[test] Pre-commit cleanup

parent a2c7f31b
...@@ -34,4 +34,5 @@ ...@@ -34,4 +34,5 @@
################################################################################### ###################################################################################
import pkg_resources import pkg_resources
prefix = pkg_resources.resource_filename(__name__, 'prefix')
prefix = pkg_resources.resource_filename(__name__, "prefix")
...@@ -38,42 +38,39 @@ import logging ...@@ -38,42 +38,39 @@ import logging
from collections import namedtuple from collections import namedtuple
from ..data import DataSource
from ..data import DataSink from ..data import DataSink
from ..data import DataSource
class MockDataSource(DataSource): class MockDataSource(DataSource):
def __init__(self, data_and_indices): def __init__(self, data_and_indices):
super(MockDataSource, self).__init__() super(MockDataSource, self).__init__()
self.infos = [ (x[1], x[2]) for x in data_and_indices ] self.infos = [(x[1], x[2]) for x in data_and_indices]
self.data = [ x[0] for x in data_and_indices ] self.data = [x[0] for x in data_and_indices]
def __getitem__(self, index): def __getitem__(self, index):
return (self.data[index], self.infos[index][0], self.infos[index][1]) return (self.data[index], self.infos[index][0], self.infos[index][1])
#---------------------------------------------------------- # ----------------------------------------------------------
class CrashingDataSource(DataSource): class CrashingDataSource(DataSource):
def __init__(self): def __init__(self):
super(CrashingDataSource, self).__init__() super(CrashingDataSource, self).__init__()
Infos = namedtuple('Infos', ['start_index', 'end_index']) Infos = namedtuple("Infos", ["start_index", "end_index"])
self.infos = [Infos(0, 0)] self.infos = [Infos(0, 0)]
def __getitem__(self, index): def __getitem__(self, index):
a = b a = b # noqa
return (None, None, None) return (None, None, None)
#---------------------------------------------------------- # ----------------------------------------------------------
class MockDataSink(DataSink): class MockDataSink(DataSink):
class WrittenData: class WrittenData:
def __init__(self, data, start, end): def __init__(self, data, start, end):
self.data = data self.data = data
...@@ -86,7 +83,8 @@ class MockDataSink(DataSink): ...@@ -86,7 +83,8 @@ class MockDataSink(DataSink):
self.dataformat = dataformat self.dataformat = dataformat
def write(self, data, start_data_index, end_data_index): def write(self, data, start_data_index, end_data_index):
if not(self.can_write): raise IOError if not (self.can_write):
raise IOError
self.written.append( self.written.append(
MockDataSink.WrittenData(data, start_data_index, end_data_index) MockDataSink.WrittenData(data, start_data_index, end_data_index)
) )
...@@ -94,6 +92,7 @@ class MockDataSink(DataSink): ...@@ -94,6 +92,7 @@ class MockDataSink(DataSink):
def isConnected(self): def isConnected(self):
return True return True
# Based on https://stackoverflow.com/a/20553331/5843716 # Based on https://stackoverflow.com/a/20553331/5843716
class MockLoggingHandler(logging.Handler): class MockLoggingHandler(logging.Handler):
"""Mock logging handler to check for expected logs. """Mock logging handler to check for expected logs.
...@@ -104,9 +103,12 @@ class MockLoggingHandler(logging.Handler): ...@@ -104,9 +103,12 @@ class MockLoggingHandler(logging.Handler):
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
self.messages = { self.messages = {
'debug': [], 'info': [], "debug": [],
'warning': [], 'error': [], "info": [],
'critical': [], 'extra': [] "warning": [],
"error": [],
"critical": [],
"extra": [],
} }
super(MockLoggingHandler, self).__init__(*args, **kwargs) super(MockLoggingHandler, self).__init__(*args, **kwargs)
......
...@@ -34,34 +34,31 @@ ...@@ -34,34 +34,31 @@
################################################################################### ###################################################################################
import unittest
import six
import os
import glob import glob
import tempfile
import logging import logging
import os
import tempfile
import unittest
import numpy as np import numpy as np
import six
from ..algorithm import Algorithm from ..algorithm import Algorithm
from ..data_loaders import DataLoaderList
from ..data_loaders import DataLoader
from ..dataformat import DataFormat
from ..data import CachedDataSink from ..data import CachedDataSink
from ..data import CachedDataSource from ..data import CachedDataSource
from ..data_loaders import DataLoader
from ..data_loaders import DataLoaderList
from ..dataformat import DataFormat
from ..inputs import Input from ..inputs import Input
from ..inputs import InputGroup from ..inputs import InputGroup
from ..inputs import InputList from ..inputs import InputList
from ..outputs import Output from ..outputs import Output
from ..outputs import OutputList from ..outputs import OutputList
from ..outputs import SynchronizationListener from ..outputs import SynchronizationListener
from . import prefix
from .mocks import MockDataSink from .mocks import MockDataSink
from .mocks import MockLoggingHandler from .mocks import MockLoggingHandler
from . import prefix
# ---------------------------------------------------------- # ----------------------------------------------------------
......
...@@ -34,24 +34,22 @@ ...@@ -34,24 +34,22 @@
################################################################################### ###################################################################################
import unittest
import os
import glob import glob
import os
import pickle # nosec import pickle # nosec
import tempfile
import shutil import shutil
import tempfile
import unittest
from ..data import mixDataIndices
from ..data import CachedDataSource
from ..data import CachedDataSink from ..data import CachedDataSink
from ..data import getAllFilenames from ..data import CachedDataSource
from ..data import foundSplitRanges from ..data import foundSplitRanges
from ..dataformat import DataFormat from ..data import getAllFilenames
from ..data import mixDataIndices
from ..database import Database from ..database import Database
from ..dataformat import DataFormat
from . import prefix from . import prefix
# ---------------------------------------------------------- # ----------------------------------------------------------
......
...@@ -34,21 +34,20 @@ ...@@ -34,21 +34,20 @@
################################################################################### ###################################################################################
import unittest
import os
import glob import glob
import os
import tempfile import tempfile
import unittest
import numpy as np import numpy as np
from ..data import CachedDataSink
from ..data import CachedDataSource
from ..data_loaders import DataLoader from ..data_loaders import DataLoader
from ..data_loaders import DataLoaderList from ..data_loaders import DataLoaderList
from ..dataformat import DataFormat from ..dataformat import DataFormat
from ..data import CachedDataSink
from ..data import CachedDataSource
from . import prefix from . import prefix
# ---------------------------------------------------------- # ----------------------------------------------------------
......
...@@ -35,11 +35,10 @@ ...@@ -35,11 +35,10 @@
import nose.tools import nose.tools
from ..database import Database
from ..database import Database
from . import prefix from . import prefix
INTEGERS_DBS = ["integers_db/{}".format(i) for i in range(1, 3)] INTEGERS_DBS = ["integers_db/{}".format(i) for i in range(1, 3)]
......
...@@ -34,20 +34,17 @@ ...@@ -34,20 +34,17 @@
################################################################################### ###################################################################################
import unittest
import tempfile
import shutil
import os import os
import shutil
import tempfile
import unittest
from ddt import ddt from ddt import ddt
from ddt import idata from ddt import idata
from ..database import Database from ..database import Database
from .test_database import INTEGERS_DBS
from . import prefix from . import prefix
from .test_database import INTEGERS_DBS
# ---------------------------------------------------------- # ----------------------------------------------------------
......
...@@ -36,24 +36,20 @@ ...@@ -36,24 +36,20 @@
# Tests for experiment execution # Tests for experiment execution
import os
import unittest
import multiprocessing import multiprocessing
import tempfile import os
import shutil import shutil
import tempfile
import unittest
from ddt import ddt from ddt import ddt
from ddt import idata from ddt import idata
from ..scripts import index
from ..hash import hashDataset from ..hash import hashDataset
from ..hash import toPath from ..hash import toPath
from ..scripts import index
from .test_database import INTEGERS_DBS
from . import prefix from . import prefix
from .test_database import INTEGERS_DBS
# ---------------------------------------------------------- # ----------------------------------------------------------
......
...@@ -36,30 +36,27 @@ ...@@ -36,30 +36,27 @@
# Tests for experiment execution # Tests for experiment execution
import os
import socket
import unittest
import json import json
import multiprocessing import multiprocessing
import tempfile import os
import shutil import shutil
import socket
import tempfile
import unittest
from contextlib import closing
import zmq import zmq
from ddt import ddt from ddt import ddt
from ddt import idata from ddt import idata
from contextlib import closing
from ..scripts import databases_provider
from ..database import Database
from ..data import RemoteDataSource from ..data import RemoteDataSource
from ..data import RemoteException from ..data import RemoteException
from ..database import Database
from .test_database import INTEGERS_DBS from ..scripts import databases_provider
from . import prefix from . import prefix
from .test_database import INTEGERS_DBS
# ---------------------------------------------------------- # ----------------------------------------------------------
......
...@@ -37,26 +37,24 @@ ...@@ -37,26 +37,24 @@
# Tests for experiment execution # Tests for experiment execution
import os import os
import shutil
import tempfile
import unittest import unittest
import zmq import zmq
import tempfile
import shutil
from ddt import ddt from ddt import ddt
from ddt import idata from ddt import idata
from ..data import RemoteDataSource
from ..data_loaders import DataLoader
from ..database import Database
from ..execution import DBExecutor from ..execution import DBExecutor
from ..execution import MessageHandler from ..execution import MessageHandler
from ..database import Database
from ..data_loaders import DataLoader
from ..data import RemoteDataSource
from ..hash import hashDataset from ..hash import hashDataset
from ..hash import toPath from ..hash import toPath
from .test_database import INTEGERS_DBS
from . import prefix from . import prefix
from .test_database import INTEGERS_DBS
# ---------------------------------------------------------- # ----------------------------------------------------------
......
...@@ -34,55 +34,41 @@ ...@@ -34,55 +34,41 @@
################################################################################### ###################################################################################
import unittest
import logging import logging
import tempfile
import simplejson
import os import os
import zmq
import shutil import shutil
import numpy as np import tempfile
import unittest
from copy import deepcopy from copy import deepcopy
from ..execution import AlgorithmExecutor import numpy as np
from ..execution import MessageHandler import simplejson
from ..inputs import InputList import zmq
from ..algorithm import Algorithm from ..algorithm import Algorithm
from ..dataformat import DataFormat
from ..data import CachedDataSink from ..data import CachedDataSink
from ..data import CachedDataSource from ..data import CachedDataSource
from ..dataformat import DataFormat
from ..execution import AlgorithmExecutor
from ..execution import MessageHandler
from ..helpers import convert_experiment_configuration_to_container from ..helpers import convert_experiment_configuration_to_container
from ..helpers import AccessMode
from . import prefix from . import prefix
from .mocks import MockLoggingHandler from .mocks import MockLoggingHandler
CONFIGURATION = { CONFIGURATION = {
'algorithm': '', "algorithm": "",
'channel': 'main', "channel": "main",
'parameters': { "parameters": {},
}, "inputs": {"in": {"path": "INPUT", "channel": "main"}},
'inputs': { "outputs": {"out": {"path": "OUTPUT", "channel": "main"}},
'in': {
'path': 'INPUT',
'channel': 'main',
}
},
'outputs': {
'out': {
'path': 'OUTPUT',
'channel': 'main'
}
},
} }
#---------------------------------------------------------- # ----------------------------------------------------------
class TestAlgorithmExecutor(unittest.TestCase): class TestAlgorithmExecutor(unittest.TestCase):
def setUp(self): def setUp(self):
self.cache_root = tempfile.mkdtemp(prefix=__name__) self.cache_root = tempfile.mkdtemp(prefix=__name__)
self.working_dir = tempfile.mkdtemp(prefix=__name__) self.working_dir = tempfile.mkdtemp(prefix=__name__)
...@@ -90,7 +76,6 @@ class TestAlgorithmExecutor(unittest.TestCase): ...@@ -90,7 +76,6 @@ class TestAlgorithmExecutor(unittest.TestCase):
self.executor_socket = None self.executor_socket = None
self.zmq_context = None self.zmq_context = None
def tearDown(self): def tearDown(self):
shutil.rmtree(self.cache_root) shutil.rmtree(self.cache_root)
shutil.rmtree(self.working_dir) shutil.rmtree(self.working_dir)
...@@ -108,15 +93,18 @@ class TestAlgorithmExecutor(unittest.TestCase): ...@@ -108,15 +93,18 @@ class TestAlgorithmExecutor(unittest.TestCase):
self.executor_socket = None self.executor_socket = None
self.zmq_context = None self.zmq_context = None
def writeData(self, input_name, indices, start_value): def writeData(self, input_name, indices, start_value):
filename = os.path.join(self.cache_root, CONFIGURATION['inputs'][input_name]['path'] + '.data') filename = os.path.join(
self.cache_root, CONFIGURATION["inputs"][input_name]["path"] + ".data"
)
dataformat = DataFormat(prefix, 'user/single_integer/1') dataformat = DataFormat(prefix, "user/single_integer/1")
self.assertTrue(dataformat.valid) self.assertTrue(dataformat.valid)
data_sink = CachedDataSink() data_sink = CachedDataSink()
self.assertTrue(data_sink.setup(filename, dataformat, indices[0][0], indices[-1][1])) self.assertTrue(
data_sink.setup(filename, dataformat, indices[0][0], indices[-1][1])
)
for i in indices: for i in indices:
data = dataformat.type() data = dataformat.type()
...@@ -130,26 +118,25 @@ class TestAlgorithmExecutor(unittest.TestCase): ...@@ -130,26 +118,25 @@ class TestAlgorithmExecutor(unittest.TestCase):
data_sink.close() data_sink.close()
del data_sink del data_sink
def process(self, algorithm_name): def process(self, algorithm_name):
self.writeData('in', [(0, 0), (1, 1), (2, 2), (3, 3)], 1000) self.writeData("in", [(0, 0), (1, 1), (2, 2), (3, 3)], 1000)
config = deepcopy(CONFIGURATION) config = deepcopy(CONFIGURATION)
config['algorithm'] = algorithm_name config["algorithm"] = algorithm_name
config = convert_experiment_configuration_to_container(config) config = convert_experiment_configuration_to_container(config)
with open(os.path.join(self.working_dir, 'configuration.json'), 'wb') as f: with open(os.path.join(self.working_dir, "configuration.json"), "wb") as f:
data = simplejson.dumps(config, indent=4).encode('utf-8') data = simplejson.dumps(config, indent=4).encode("utf-8")
f.write(data) f.write(data)
working_prefix = os.path.join(self.working_dir, 'prefix') working_prefix = os.path.join(self.working_dir, "prefix")
if not os.path.exists(working_prefix): if not os.path.exists(working_prefix):
os.makedirs(working_prefix) os.makedirs(working_prefix)
algorithm = Algorithm(prefix, algorithm_name) algorithm = Algorithm(prefix, algorithm_name)
algorithm.export(working_prefix) algorithm.export(working_prefix)
self.message_handler = MessageHandler('127.0.0.1') self.message_handler = MessageHandler("127.0.0.1")
self.message_handler.start() self.message_handler.start()
...@@ -157,14 +144,23 @@ class TestAlgorithmExecutor(unittest.TestCase): ...@@ -157,14 +144,23 @@ class TestAlgorithmExecutor(unittest.TestCase):
self.executor_socket = self.zmq_context.socket(zmq.PAIR) self.executor_socket = self.zmq_context.socket(zmq.PAIR)
self.executor_socket.connect(self.message_handler.address) self.executor_socket.connect(self.message_handler.address)
executor = AlgorithmExecutor(self.executor_socket, self.working_dir, cache_root=self.cache_root) executor = AlgorithmExecutor(
self.executor_socket, self.working_dir, cache_root=self.cache_root
)
self.assertTrue(executor.setup()) self.assertTrue(executor.setup())
self.assertTrue(executor.prepare()) self.assertTrue(executor.prepare())
self.assertTrue(executor.process()) self.assertTrue(executor.process())
cached_file = CachedDataSource() cached_file = CachedDataSource()
self.assertTrue(cached_file.setup(os.path.join(self.cache_root, CONFIGURATION['outputs']['out']['path'] + '.data'), prefix)) self.assertTrue(
cached_file.setup(
os.path.join(
self.cache_root, CONFIGURATION["outputs"]["out"]["path"] + ".data"
),
prefix,
)
)
for i in range(len(cached_file)): for i in range(len(cached_file)):
data, start, end = cached_file[i] data, start, end = cached_file[i]
...@@ -172,22 +168,22 @@ class TestAlgorithmExecutor(unittest.TestCase): ...@@ -172,22 +168,22 @@ class TestAlgorithmExecutor(unittest.TestCase):
self.assertEqual(start, i) self.assertEqual(start, i)
self.assertEqual(end, i) self.assertEqual(end, i)
def test_legacy_echo_1(self): def test_legacy_echo_1(self):
log_handler = MockLoggingHandler(level='DEBUG') log_handler = MockLoggingHandler(level="DEBUG")
logging.getLogger().addHandler(log_handler) logging.getLogger().addHandler(log_handler)
log_messages = log_handler.messages log_messages = log_handler.messages