Commit 39e23b21 authored by Samuel GAIST's avatar Samuel GAIST

[test] Pre-commit cleanup

parent a2c7f31b
......@@ -34,4 +34,5 @@
###################################################################################
import pkg_resources
prefix = pkg_resources.resource_filename(__name__, 'prefix')
prefix = pkg_resources.resource_filename(__name__, "prefix")
......@@ -38,42 +38,39 @@ import logging
from collections import namedtuple
from ..data import DataSource
from ..data import DataSink
from ..data import DataSource
class MockDataSource(DataSource):
def __init__(self, data_and_indices):
super(MockDataSource, self).__init__()
self.infos = [ (x[1], x[2]) for x in data_and_indices ]
self.data = [ x[0] for x in data_and_indices ]
self.infos = [(x[1], x[2]) for x in data_and_indices]
self.data = [x[0] for x in data_and_indices]
def __getitem__(self, index):
return (self.data[index], self.infos[index][0], self.infos[index][1])
#----------------------------------------------------------
# ----------------------------------------------------------
class CrashingDataSource(DataSource):
def __init__(self):
super(CrashingDataSource, self).__init__()
Infos = namedtuple('Infos', ['start_index', 'end_index'])
Infos = namedtuple("Infos", ["start_index", "end_index"])
self.infos = [Infos(0, 0)]
def __getitem__(self, index):
a = b
a = b # noqa
return (None, None, None)
#----------------------------------------------------------
# ----------------------------------------------------------
class MockDataSink(DataSink):
class WrittenData:
def __init__(self, data, start, end):
self.data = data
......@@ -86,7 +83,8 @@ class MockDataSink(DataSink):
self.dataformat = dataformat
def write(self, data, start_data_index, end_data_index):
if not(self.can_write): raise IOError
if not (self.can_write):
raise IOError
self.written.append(
MockDataSink.WrittenData(data, start_data_index, end_data_index)
)
......@@ -94,6 +92,7 @@ class MockDataSink(DataSink):
def isConnected(self):
return True
# Based on https://stackoverflow.com/a/20553331/5843716
class MockLoggingHandler(logging.Handler):
"""Mock logging handler to check for expected logs.
......@@ -104,9 +103,12 @@ class MockLoggingHandler(logging.Handler):
def __init__(self, *args, **kwargs):
self.messages = {
'debug': [], 'info': [],
'warning': [], 'error': [],
'critical': [], 'extra': []
"debug": [],
"info": [],
"warning": [],
"error": [],
"critical": [],
"extra": [],
}
super(MockLoggingHandler, self).__init__(*args, **kwargs)
......
......@@ -34,34 +34,31 @@
###################################################################################
import unittest
import six
import os
import glob
import tempfile
import logging
import os
import tempfile
import unittest
import numpy as np
import six
from ..algorithm import Algorithm
from ..data_loaders import DataLoaderList
from ..data_loaders import DataLoader
from ..dataformat import DataFormat
from ..data import CachedDataSink
from ..data import CachedDataSource
from ..data_loaders import DataLoader
from ..data_loaders import DataLoaderList
from ..dataformat import DataFormat
from ..inputs import Input
from ..inputs import InputGroup
from ..inputs import InputList
from ..outputs import Output
from ..outputs import OutputList
from ..outputs import SynchronizationListener
from . import prefix
from .mocks import MockDataSink
from .mocks import MockLoggingHandler
from . import prefix
# ----------------------------------------------------------
......
......@@ -34,24 +34,22 @@
###################################################################################
import unittest
import os
import glob
import os
import pickle # nosec
import tempfile
import shutil
import tempfile
import unittest
from ..data import mixDataIndices
from ..data import CachedDataSource
from ..data import CachedDataSink
from ..data import getAllFilenames
from ..data import CachedDataSource
from ..data import foundSplitRanges
from ..dataformat import DataFormat
from ..data import getAllFilenames
from ..data import mixDataIndices
from ..database import Database
from ..dataformat import DataFormat
from . import prefix
# ----------------------------------------------------------
......
......@@ -34,21 +34,20 @@
###################################################################################
import unittest
import os
import glob
import os
import tempfile
import unittest
import numpy as np
from ..data import CachedDataSink
from ..data import CachedDataSource
from ..data_loaders import DataLoader
from ..data_loaders import DataLoaderList
from ..dataformat import DataFormat
from ..data import CachedDataSink
from ..data import CachedDataSource
from . import prefix
# ----------------------------------------------------------
......
......@@ -35,11 +35,10 @@
import nose.tools
from ..database import Database
from ..database import Database
from . import prefix
INTEGERS_DBS = ["integers_db/{}".format(i) for i in range(1, 3)]
......
......@@ -34,20 +34,17 @@
###################################################################################
import unittest
import tempfile
import shutil
import os
import shutil
import tempfile
import unittest
from ddt import ddt
from ddt import idata
from ..database import Database
from .test_database import INTEGERS_DBS
from . import prefix
from .test_database import INTEGERS_DBS
# ----------------------------------------------------------
......
......@@ -36,24 +36,20 @@
# Tests for experiment execution
import os
import unittest
import multiprocessing
import tempfile
import os
import shutil
import tempfile
import unittest
from ddt import ddt
from ddt import idata
from ..scripts import index
from ..hash import hashDataset
from ..hash import toPath
from .test_database import INTEGERS_DBS
from ..scripts import index
from . import prefix
from .test_database import INTEGERS_DBS
# ----------------------------------------------------------
......
......@@ -36,30 +36,27 @@
# Tests for experiment execution
import os
import socket
import unittest
import json
import multiprocessing
import tempfile
import os
import shutil
import socket
import tempfile
import unittest
from contextlib import closing
import zmq
from ddt import ddt
from ddt import idata
from contextlib import closing
from ..scripts import databases_provider
from ..database import Database
from ..data import RemoteDataSource
from ..data import RemoteException
from .test_database import INTEGERS_DBS
from ..database import Database
from ..scripts import databases_provider
from . import prefix
from .test_database import INTEGERS_DBS
# ----------------------------------------------------------
......
......@@ -37,26 +37,24 @@
# Tests for experiment execution
import os
import shutil
import tempfile
import unittest
import zmq
import tempfile
import shutil
from ddt import ddt
from ddt import idata
from ..data import RemoteDataSource
from ..data_loaders import DataLoader
from ..database import Database
from ..execution import DBExecutor
from ..execution import MessageHandler
from ..database import Database
from ..data_loaders import DataLoader
from ..data import RemoteDataSource
from ..hash import hashDataset
from ..hash import toPath
from .test_database import INTEGERS_DBS
from . import prefix
from .test_database import INTEGERS_DBS
# ----------------------------------------------------------
......
......@@ -34,55 +34,41 @@
###################################################################################
import unittest
import logging
import tempfile
import simplejson
import os
import zmq
import shutil
import numpy as np
import tempfile
import unittest
from copy import deepcopy
from ..execution import AlgorithmExecutor
from ..execution import MessageHandler
from ..inputs import InputList
import numpy as np
import simplejson
import zmq
from ..algorithm import Algorithm
from ..dataformat import DataFormat
from ..data import CachedDataSink
from ..data import CachedDataSource
from ..dataformat import DataFormat
from ..execution import AlgorithmExecutor
from ..execution import MessageHandler
from ..helpers import convert_experiment_configuration_to_container
from ..helpers import AccessMode
from . import prefix
from .mocks import MockLoggingHandler
CONFIGURATION = {
'algorithm': '',
'channel': 'main',
'parameters': {
},
'inputs': {
'in': {
'path': 'INPUT',
'channel': 'main',
}
},
'outputs': {
'out': {
'path': 'OUTPUT',
'channel': 'main'
}
},
"algorithm": "",
"channel": "main",
"parameters": {},
"inputs": {"in": {"path": "INPUT", "channel": "main"}},
"outputs": {"out": {"path": "OUTPUT", "channel": "main"}},
}
#----------------------------------------------------------
# ----------------------------------------------------------
class TestAlgorithmExecutor(unittest.TestCase):
def setUp(self):
self.cache_root = tempfile.mkdtemp(prefix=__name__)
self.working_dir = tempfile.mkdtemp(prefix=__name__)
......@@ -90,7 +76,6 @@ class TestAlgorithmExecutor(unittest.TestCase):
self.executor_socket = None
self.zmq_context = None
def tearDown(self):
shutil.rmtree(self.cache_root)
shutil.rmtree(self.working_dir)
......@@ -108,15 +93,18 @@ class TestAlgorithmExecutor(unittest.TestCase):
self.executor_socket = None
self.zmq_context = None
def writeData(self, input_name, indices, start_value):
filename = os.path.join(self.cache_root, CONFIGURATION['inputs'][input_name]['path'] + '.data')
filename = os.path.join(
self.cache_root, CONFIGURATION["inputs"][input_name]["path"] + ".data"
)
dataformat = DataFormat(prefix, 'user/single_integer/1')
dataformat = DataFormat(prefix, "user/single_integer/1")
self.assertTrue(dataformat.valid)
data_sink = CachedDataSink()
self.assertTrue(data_sink.setup(filename, dataformat, indices[0][0], indices[-1][1]))
self.assertTrue(
data_sink.setup(filename, dataformat, indices[0][0], indices[-1][1])
)
for i in indices:
data = dataformat.type()
......@@ -130,26 +118,25 @@ class TestAlgorithmExecutor(unittest.TestCase):
data_sink.close()
del data_sink
def process(self, algorithm_name):
self.writeData('in', [(0, 0), (1, 1), (2, 2), (3, 3)], 1000)
self.writeData("in", [(0, 0), (1, 1), (2, 2), (3, 3)], 1000)
config = deepcopy(CONFIGURATION)
config['algorithm'] = algorithm_name
config["algorithm"] = algorithm_name
config = convert_experiment_configuration_to_container(config)
with open(os.path.join(self.working_dir, 'configuration.json'), 'wb') as f:
data = simplejson.dumps(config, indent=4).encode('utf-8')
with open(os.path.join(self.working_dir, "configuration.json"), "wb") as f:
data = simplejson.dumps(config, indent=4).encode("utf-8")
f.write(data)
working_prefix = os.path.join(self.working_dir, 'prefix')
working_prefix = os.path.join(self.working_dir, "prefix")
if not os.path.exists(working_prefix):
os.makedirs(working_prefix)
algorithm = Algorithm(prefix, algorithm_name)
algorithm.export(working_prefix)
self.message_handler = MessageHandler('127.0.0.1')
self.message_handler = MessageHandler("127.0.0.1")
self.message_handler.start()
......@@ -157,14 +144,23 @@ class TestAlgorithmExecutor(unittest.TestCase):
self.executor_socket = self.zmq_context.socket(zmq.PAIR)
self.executor_socket.connect(self.message_handler.address)
executor = AlgorithmExecutor(self.executor_socket, self.working_dir, cache_root=self.cache_root)
executor = AlgorithmExecutor(
self.executor_socket, self.working_dir, cache_root=self.cache_root
)
self.assertTrue(executor.setup())
self.assertTrue(executor.prepare())
self.assertTrue(executor.process())
cached_file = CachedDataSource()
self.assertTrue(cached_file.setup(os.path.join(self.cache_root, CONFIGURATION['outputs']['out']['path'] + '.data'), prefix))
self.assertTrue(
cached_file.setup(
os.path.join(
self.cache_root, CONFIGURATION["outputs"]["out"]["path"] + ".data"
),
prefix,
)
)
for i in range(len(cached_file)):
data, start, end = cached_file[i]
......@@ -172,22 +168,22 @@ class TestAlgorithmExecutor(unittest.TestCase):
self.assertEqual(start, i)
self.assertEqual(end, i)
def test_legacy_echo_1(self):
log_handler = MockLoggingHandler(level='DEBUG')
log_handler = MockLoggingHandler(level="DEBUG")
logging.getLogger().addHandler(log_handler)
log_messages = log_handler.messages
self.process('legacy/echo/1')
self.process("legacy/echo/1")
info_len = len(log_messages['warning'])
info_len = len(log_messages["warning"])
self.assertEqual(info_len, 1)
self.assertEqual(log_messages['warning'][info_len - 1], 'legacy/echo/1 is using LEGACY I/O API, please upgrade this algorithm as soon as possible')
self.assertEqual(
log_messages["warning"][info_len - 1],
"legacy/echo/1 is using LEGACY I/O API, please upgrade this algorithm as soon as possible",
)
def test_sequential_echo_1(self):
self.process('sequential/echo/1')
self.process("sequential/echo/1")
def test_autonomous_echo_1(self):
self.process('autonomous/echo/1')
self.process("autonomous/echo/1")
......@@ -34,14 +34,12 @@
###################################################################################
import numpy
import nose.tools
import numpy
from ..dataformat import DataFormat
from . import prefix
# ----------------------------------------------------------
......
......@@ -34,14 +34,12 @@
###################################################################################
import numpy
import nose.tools
import numpy
from ..dataformat import DataFormat
from . import prefix
# ----------------------------------------------------------
......
......@@ -37,10 +37,8 @@
import nose.tools
from ..dataformat import DataFormat
from . import prefix
# ----------------------------------------------------------
......
......@@ -34,14 +34,12 @@
###################################################################################
import numpy
import nose.tools
import numpy
from ..dataformat import DataFormat
from . import prefix
# ----------------------------------------------------------
......
......@@ -34,14 +34,12 @@
###################################################################################
import numpy
import nose.tools
import numpy
from ..dataformat import DataFormat
from . import prefix
# ----------------------------------------------------------
......
......@@ -34,14 +34,12 @@
###################################################################################
import six
import nose.tools
import six
from ..dataformat import DataFormat
from . import prefix
number42L = long(42) if six.PY2 else int(42) # noqa
......
......@@ -34,14 +34,12 @@
###################################################################################
import numpy
import nose.tools
import numpy
from ..dataformat import DataFormat
from . import prefix
# ----------------------------------------------------------
......
......@@ -39,7 +39,6 @@ import six
from .. import hash
# ----------------------------------------------------------
......
......@@ -37,22 +37,21 @@
# Tests for experiment execution
import os
import shutil
import tempfile
import unittest
import zmq
import tempfile
import shutil
from ..algorithm import Algorithm
from ..database import Database
from ..execution import MessageHandler
from ..hash import hashDataset
from ..hash import toPath
from ..algorithm import Algorithm
from ..helpers import create_inputs_from_configuration
from ..helpers import AccessMode
from ..execution import MessageHandler
from ..helpers import create_inputs_from_configuration
from . import prefix
# ----------------------------------------------------------
......
......@@ -36,66 +36,59 @@
import unittest
from .mocks import MockDataSource
from ..outputs import SynchronizationListener
from ..dataformat import DataFormat
from ..inputs import Input
from ..inputs import InputGroup
from ..inputs import InputList
from ..dataformat import DataFormat
from ..data import CachedDataSink
from ..outputs import SynchronizationListener
from . import prefix
from .mocks import MockDataSource
#----------------------------------------------------------
# ----------------------------------------------------------
class InputTest(unittest.TestCase):
def test_creation(self):
data_source = MockDataSource([])
input = Input('test', 'mock', data_source)