Commit 928a2797 authored by Flavio TARSETTI's avatar Flavio TARSETTI

Merge branch 'logger_cleanup' into 'master'

Logger cleanup

See merge request !69
parents 1cdad443 fb229608
Pipeline #38028 failed with stages
in 90 minutes
...@@ -415,7 +415,7 @@ class CachedDataSource(DataSource): ...@@ -415,7 +415,7 @@ class CachedDataSource(DataSource):
) = getAllFilenames(filename, start_index, end_index) ) = getAllFilenames(filename, start_index, end_index)
if len(self.filenames) == 0: if len(self.filenames) == 0:
logger.warn("No files found for %s" % filename) logger.warning("No files found for %s" % filename)
return False return False
check_consistency(self.filenames, data_checksum_filenames) check_consistency(self.filenames, data_checksum_filenames)
...@@ -429,7 +429,7 @@ class CachedDataSource(DataSource): ...@@ -429,7 +429,7 @@ class CachedDataSource(DataSource):
try: try:
f = open(current_filename, "rb") f = open(current_filename, "rb")
except Exception as e: except Exception as e:
logger.warn("Could not setup `%s': %s" % (filename, e)) logger.warning("Could not setup `%s': %s" % (filename, e))
return False return False
# Reads the header of the current file # Reads the header of the current file
......
...@@ -42,12 +42,10 @@ data_loaders ...@@ -42,12 +42,10 @@ data_loaders
This module implements all the data communication related classes This module implements all the data communication related classes
""" """
import logging
import six import six
from .data import mixDataIndices from .data import mixDataIndices
logger = logging.getLogger(__name__)
# ---------------------------------------------------------- # ----------------------------------------------------------
...@@ -90,35 +88,36 @@ class DataView(object): ...@@ -90,35 +88,36 @@ class DataView(object):
""" """
def __init__(self, data_loader, data_indices): def __init__(self, data_loader, data_indices):
self.infos = {} self.infos = {}
self.data_indices = data_indices self.data_indices = data_indices
self.nb_data_units = len(data_indices) self.nb_data_units = len(data_indices)
self.data_index_start = data_indices[0][0] self.data_index_start = data_indices[0][0]
self.data_index_end = data_indices[-1][1] self.data_index_end = data_indices[-1][1]
for input_name, infos in data_loader.infos.items(): for input_name, infos in data_loader.infos.items():
input_data_indices = [] input_data_indices = []
current_start = self.data_index_start current_start = self.data_index_start
for i in range(self.data_index_start, self.data_index_end + 1): for i in range(self.data_index_start, self.data_index_end + 1):
for indices in infos['data_indices']: for indices in infos["data_indices"]:
if indices[1] == i: if indices[1] == i:
input_data_indices.append( (current_start, i) ) input_data_indices.append((current_start, i))
current_start = i + 1 current_start = i + 1
break break
if (len(input_data_indices) == 0) or (input_data_indices[-1][1] != self.data_index_end): if (len(input_data_indices) == 0) or (
input_data_indices.append( (current_start, self.data_index_end) ) input_data_indices[-1][1] != self.data_index_end
):
input_data_indices.append((current_start, self.data_index_end))
self.infos[input_name] = dict( self.infos[input_name] = dict(
data_source = infos['data_source'], data_source=infos["data_source"],
data_indices = input_data_indices, data_indices=input_data_indices,
data = None, data=None,
start_index = -1, start_index=-1,
end_index = -1, end_index=-1,
) )
def count(self, input_name=None): def count(self, input_name=None):
"""Returns the number of available data indexes for the given input """Returns the number of available data indexes for the given input
name. If none given the number of available data units. name. If none given the number of available data units.
...@@ -134,30 +133,30 @@ class DataView(object): ...@@ -134,30 +133,30 @@ class DataView(object):
""" """
if input_name is not None: if input_name is not None:
try: try:
return len(self.infos[input_name]['data_indices']) return len(self.infos[input_name]["data_indices"])
except: except Exception:
return None return None
else: else:
return self.nb_data_units return self.nb_data_units
def __getitem__(self, index): def __getitem__(self, index):
if index < 0: if index < 0:
return (None, None, None) return (None, None, None)
try: try:
indices = self.data_indices[index] indices = self.data_indices[index]
except: except Exception:
return (None, None, None) return (None, None, None)
result = {} result = {}
for input_name, infos in self.infos.items(): for input_name, infos in self.infos.items():
if (indices[0] < infos['start_index']) or (infos['end_index'] < indices[0]): if (indices[0] < infos["start_index"]) or (infos["end_index"] < indices[0]):
(infos['data'], infos['start_index'], infos['end_index']) = \ (infos["data"], infos["start_index"], infos["end_index"]) = infos[
infos['data_source'].getAtDataIndex(indices[0]) "data_source"
].getAtDataIndex(indices[0])
result[input_name] = infos['data'] result[input_name] = infos["data"]
return (result, indices[0], indices[1]) return (result, indices[0], indices[1])
...@@ -204,35 +203,34 @@ class DataLoader(object): ...@@ -204,35 +203,34 @@ class DataLoader(object):
""" """
def __init__(self, channel): def __init__(self, channel):
self.channel = str(channel) self.channel = str(channel)
self.infos = {} self.infos = {}
self.mixed_data_indices = None self.mixed_data_indices = None
self.nb_data_units = 0 self.nb_data_units = 0
self.data_index_start = -1 # Lower index across all inputs self.data_index_start = -1 # Lower index across all inputs
self.data_index_end = -1 # Bigger index across all inputs self.data_index_end = -1 # Bigger index across all inputs
def add(self, input_name, data_source): def add(self, input_name, data_source):
self.infos[input_name] = dict( self.infos[input_name] = dict(
data_source = data_source, data_source=data_source,
data_indices = data_source.data_indices(), data_indices=data_source.data_indices(),
data = None, data=None,
start_index = -1, start_index=-1,
end_index = -1, end_index=-1,
) )
self.mixed_data_indices = mixDataIndices([ x['data_indices'] for x in self.infos.values() ]) self.mixed_data_indices = mixDataIndices(
[x["data_indices"] for x in self.infos.values()]
)
self.nb_data_units = len(self.mixed_data_indices) self.nb_data_units = len(self.mixed_data_indices)
self.data_index_start = self.mixed_data_indices[0][0] self.data_index_start = self.mixed_data_indices[0][0]
self.data_index_end = self.mixed_data_indices[-1][1] self.data_index_end = self.mixed_data_indices[-1][1]
def input_names(self): def input_names(self):
"""Returns the name of all inputs associated to this data loader""" """Returns the name of all inputs associated to this data loader"""
return self.infos.keys() return self.infos.keys()
def count(self, input_name=None): def count(self, input_name=None):
"""Returns the number of available data indexes for the given input """Returns the number of available data indexes for the given input
name. If none given the number of available data units. name. If none given the number of available data units.
...@@ -249,13 +247,12 @@ class DataLoader(object): ...@@ -249,13 +247,12 @@ class DataLoader(object):
if input_name is not None: if input_name is not None:
try: try:
return len(self.infos[input_name]['data_indices']) return len(self.infos[input_name]["data_indices"])
except: except Exception:
return 0 return 0
else: else:
return self.nb_data_units return self.nb_data_units
def view(self, input_name, index): def view(self, input_name, index):
""" Returns the view associated with this data loader """ Returns the view associated with this data loader
...@@ -272,33 +269,36 @@ class DataLoader(object): ...@@ -272,33 +269,36 @@ class DataLoader(object):
return None return None
try: try:
indices = self.infos[input_name]['data_indices'][index] indices = self.infos[input_name]["data_indices"][index]
except: except Exception:
return None return None
limited_data_indices = [ x for x in self.mixed_data_indices limited_data_indices = [
if (indices[0] <= x[0]) and (x[1] <= indices[1]) ] x
for x in self.mixed_data_indices
if (indices[0] <= x[0]) and (x[1] <= indices[1])
]
return DataView(self, limited_data_indices) return DataView(self, limited_data_indices)
def __getitem__(self, index): def __getitem__(self, index):
if index < 0: if index < 0:
return (None, None, None) return (None, None, None)
try: try:
indices = self.mixed_data_indices[index] indices = self.mixed_data_indices[index]
except: except Exception:
return (None, None, None) return (None, None, None)
result = {} result = {}
for input_name, infos in self.infos.items(): for input_name, infos in self.infos.items():
if (indices[0] < infos['start_index']) or (infos['end_index'] < indices[0]): if (indices[0] < infos["start_index"]) or (infos["end_index"] < indices[0]):
(infos['data'], infos['start_index'], infos['end_index']) = \ (infos["data"], infos["start_index"], infos["end_index"]) = infos[
infos['data_source'].getAtDataIndex(indices[0]) "data_source"
].getAtDataIndex(indices[0])
result[input_name] = infos['data'] result[input_name] = infos["data"]
return (result, indices[0], indices[1]) return (result, indices[0], indices[1])
...@@ -354,7 +354,6 @@ class DataLoaderList(object): ...@@ -354,7 +354,6 @@ class DataLoaderList(object):
self._loaders = [] self._loaders = []
self.main_loader = None self.main_loader = None
def add(self, data_loader): def add(self, data_loader):
"""Add a data loader to the list """Add a data loader to the list
...@@ -366,7 +365,6 @@ class DataLoaderList(object): ...@@ -366,7 +365,6 @@ class DataLoaderList(object):
self._loaders.append(data_loader) self._loaders.append(data_loader)
def __getitem__(self, name_or_index): def __getitem__(self, name_or_index):
try: try:
if isinstance(name_or_index, six.string_types): if isinstance(name_or_index, six.string_types):
...@@ -374,30 +372,24 @@ class DataLoaderList(object): ...@@ -374,30 +372,24 @@ class DataLoaderList(object):
elif isinstance(name_or_index, int): elif isinstance(name_or_index, int):
return self._loaders[name_or_index] return self._loaders[name_or_index]
except: except Exception:
pass return None
return None
def __iter__(self): def __iter__(self):
for i in range(len(self._loaders)): for i in range(len(self._loaders)):
yield self._loaders[i] yield self._loaders[i]
def __len__(self): def __len__(self):
return len(self._loaders) return len(self._loaders)
def loaderOf(self, input_name): def loaderOf(self, input_name):
"""Returns the data loader matching the input name""" """Returns the data loader matching the input name"""
try: try:
return [ k for k in self._loaders if input_name in k.input_names() ][0] return [k for k in self._loaders if input_name in k.input_names()][0]
except: except Exception:
return None return None
def secondaries(self): def secondaries(self):
"""Returns a list of all data loaders except the main one""" """Returns a list of all data loaders except the main one"""
......
...@@ -43,17 +43,11 @@ Execution utilities ...@@ -43,17 +43,11 @@ Execution utilities
""" """
import os import os
import logging
import simplejson import simplejson
from ..database import Database from ..database import Database
logger = logging.getLogger(__name__)
class DBExecutor(object): class DBExecutor(object):
"""Executor specialised in database views """Executor specialised in database views
......
...@@ -336,7 +336,7 @@ class MessageHandler(threading.Thread): ...@@ -336,7 +336,7 @@ class MessageHandler(threading.Thread):
answer = self.socket.recv() # ack answer = self.socket.recv() # ack
logger.debug("recv: %s", answer) logger.debug("recv: %s", answer)
break break
logger.warn( logger.warning(
'(try %d) waited %d ms for "ack" from server', this_try, timeout '(try %d) waited %d ms for "ack" from server', this_try, timeout
) )
this_try += 1 this_try += 1
......
This diff is collapsed.
...@@ -43,12 +43,8 @@ This module implements output related classes ...@@ -43,12 +43,8 @@ This module implements output related classes
""" """
import six import six
import logging
import zmq import zmq
logger = logging.getLogger(__name__)
class SynchronizationListener: class SynchronizationListener:
"""A callback mechanism to keep Inputs and Outputs in groups and lists """A callback mechanism to keep Inputs and Outputs in groups and lists
......
...@@ -95,7 +95,7 @@ def send_error(logger, socket, tp, message): ...@@ -95,7 +95,7 @@ def send_error(logger, socket, tp, message):
answer = socket.recv() # ack answer = socket.recv() # ack
logger.debug("recv: %s", answer) logger.debug("recv: %s", answer)
break break
logger.warn('(try %d) waited %d ms for "ack" from server', this_try, timeout) logger.warning('(try %d) waited %d ms for "ack" from server', this_try, timeout)
this_try += 1 this_try += 1
if this_try > max_tries: if this_try > max_tries:
logger.error("could not send error message to server") logger.error("could not send error message to server")
......
...@@ -38,7 +38,6 @@ ...@@ -38,7 +38,6 @@
import os import os
import socket import socket
import logging
import unittest import unittest
import json import json
...@@ -62,9 +61,6 @@ from .test_database import INTEGERS_DBS ...@@ -62,9 +61,6 @@ from .test_database import INTEGERS_DBS
from . import prefix from . import prefix
logger = logging.getLogger(__name__)
# ---------------------------------------------------------- # ----------------------------------------------------------
......
...@@ -37,7 +37,6 @@ ...@@ -37,7 +37,6 @@
# Tests for experiment execution # Tests for experiment execution
import os import os
import logging
import unittest import unittest
import zmq import zmq
import tempfile import tempfile
...@@ -62,9 +61,6 @@ from . import prefix ...@@ -62,9 +61,6 @@ from . import prefix
# ---------------------------------------------------------- # ----------------------------------------------------------
logger = logging.getLogger(__name__)
CONFIGURATION = { CONFIGURATION = {
"queue": "queue", "queue": "queue",
"algorithm": "user/sum/1", "algorithm": "user/sum/1",
......
This diff is collapsed.
...@@ -37,7 +37,6 @@ ...@@ -37,7 +37,6 @@
# Tests for experiment execution # Tests for experiment execution
import os import os
import logging
import unittest import unittest
import zmq import zmq
import tempfile import tempfile
...@@ -63,9 +62,6 @@ from ..helpers import convert_experiment_configuration_to_container ...@@ -63,9 +62,6 @@ from ..helpers import convert_experiment_configuration_to_container
from . import prefix from . import prefix
logger = logging.getLogger(__name__)
# ---------------------------------------------------------- # ----------------------------------------------------------
......
...@@ -34,9 +34,6 @@ ...@@ -34,9 +34,6 @@
################################################################################### ###################################################################################
import logging
logger = logging.getLogger(__name__)
import unittest import unittest
import zmq import zmq
import os import os
...@@ -46,9 +43,6 @@ import numpy as np ...@@ -46,9 +43,6 @@ import numpy as np
from ..execution import MessageHandler from ..execution import MessageHandler
from ..dataformat import DataFormat from ..dataformat import DataFormat
from ..inputs import Input
from ..inputs import InputGroup
from ..inputs import InputList
from ..data import RemoteException from ..data import RemoteException
from ..data import CachedDataSource from ..data import CachedDataSource
from ..data import RemoteDataSource from ..data import RemoteDataSource
...@@ -59,21 +53,19 @@ from .mocks import CrashingDataSource ...@@ -59,21 +53,19 @@ from .mocks import CrashingDataSource
from . import prefix from . import prefix
#---------------------------------------------------------- # ----------------------------------------------------------
class TestMessageHandlerBase(unittest.TestCase): class TestMessageHandlerBase(unittest.TestCase):
def setUp(self): def setUp(self):
self.filenames = [] self.filenames = []
self.data_loader = None self.data_loader = None
def tearDown(self): def tearDown(self):
for filename in self.filenames: for filename in self.filenames:
basename, ext = os.path.splitext(filename) basename, ext = os.path.splitext(filename)
filenames = [filename] filenames = [filename]
filenames += glob.glob(basename + '*') filenames += glob.glob(basename + "*")
for filename in filenames: for filename in filenames:
if os.path.exists(filename): if os.path.exists(filename):
os.unlink(filename) os.unlink(filename)
...@@ -87,32 +79,34 @@ class TestMessageHandlerBase(unittest.TestCase): ...@@ -87,32 +79,34 @@ class TestMessageHandlerBase(unittest.TestCase):
self.data_loader = None self.data_loader = None
def create_data_loader(self, data_sources): def create_data_loader(self, data_sources):
self.client_context = zmq.Context() self.client_context = zmq.Context()
self.message_handler = MessageHandler('127.0.0.1', data_sources=data_sources, context=self.client_context) self.message_handler = MessageHandler(
"127.0.0.1", data_sources=data_sources, context=self.client_context
)
self.message_handler.start() self.message_handler.start()
self.client_socket = self.client_context.socket(zmq.PAIR) self.client_socket = self.client_context.socket(zmq.PAIR)
self.client_socket.connect(self.message_handler.address) self.client_socket.connect(self.message_handler.address)
self.data_loader = DataLoader('channel') self.data_loader = DataLoader("channel")
for input_name in data_sources.keys(): for input_name in data_sources.keys():
data_source = RemoteDataSource() data_source = RemoteDataSource()
data_source.setup(self.client_socket, input_name, 'user/single_integer/1', prefix) data_source.setup(
self.client_socket, input_name, "user/single_integer/1", prefix
)
self.data_loader.add(input_name, data_source) self.data_loader.add(input_name, data_source)
def writeData(self, start_index=0, end_index=10, step=1, base=0): def writeData(self, start_index=0, end_index=10, step=1, base=0):
testfile = tempfile.NamedTemporaryFile(prefix=__name__, suffix='.data') testfile = tempfile.NamedTemporaryFile(prefix=__name__, suffix=".data")
testfile.close() # preserve only the name testfile.close() # preserve only the name
filename = testfile.name filename = testfile.name
self.filenames.append(filename) self.filenames.append(filename)
dataformat = DataFormat(prefix, 'user/single_integer/1') dataformat = DataFormat(prefix, "user/single_integer/1")
self.assertTrue(dataformat.valid) self.assertTrue(dataformat.valid)
data_sink = CachedDataSink() data_sink = CachedDataSink()
...@@ -140,102 +134,94 @@ class TestMessageHandlerBase(unittest.TestCase): ...@@ -140,102 +134,94 @@ class TestMessageHandlerBase(unittest.TestCase):
return cached_file return cached_file
#---------------------------------------------------------- # ----------------------------------------------------------
class TestOneDataSource(TestMessageHandlerBase): class TestOneDataSource(TestMessageHandlerBase):
def setUp(self): def setUp(self):
super(TestOneDataSource, self).setUp() super(TestOneDataSource, self).setUp()
data_sources = {} data_sources = {}
data_sources['a'] = self.writeData(start_index=0, end_index=9)