Skip to content
Snippets Groups Projects
Commit 4156bbf4 authored by Samuel GAIST's avatar Samuel GAIST
Browse files

[data] Added missing doc and fixed doc related warnings

Also fixed import warnings
parent 3f24f857
Branches
Tags
2 merge requests!17Merge development branch 1.5.x,!13Improve documentation
...@@ -26,33 +26,38 @@ ...@@ -26,33 +26,38 @@
############################################################################### ###############################################################################
"""Data I/O classes and functions""" """
====
data
====
Data I/O classes and functions
"""
import os import os
import re import re
import glob import glob
import simplejson as json import simplejson as json
import select
import time import time
import tempfile
import abc import abc
import zmq import zmq
import logging
import six
from functools import reduce from functools import reduce
from collections import namedtuple from collections import namedtuple
import logging
logger = logging.getLogger(__name__)
import six
from .hash import hashFileContents from .hash import hashFileContents
from .dataformat import DataFormat from .dataformat import DataFormat
from .algorithm import Algorithm from .algorithm import Algorithm
logger = logging.getLogger(__name__)
#---------------------------------------------------------- # ----------------------------------------------------------
class RemoteException(Exception): class RemoteException(Exception):
"""Exception happening on a remote location"""
def __init__(self, kind, message): def __init__(self, kind, message):
super(RemoteException, self).__init__() super(RemoteException, self).__init__()
...@@ -71,7 +76,7 @@ class RemoteException(Exception): ...@@ -71,7 +76,7 @@ class RemoteException(Exception):
return '(usr) {}'.format(self.user_error) return '(usr) {}'.format(self.user_error)
#---------------------------------------------------------- # ----------------------------------------------------------
def mixDataIndices(list_of_data_indices): def mixDataIndices(list_of_data_indices):
...@@ -129,7 +134,7 @@ def mixDataIndices(list_of_data_indices): ...@@ -129,7 +134,7 @@ def mixDataIndices(list_of_data_indices):
return result return result
#---------------------------------------------------------- # ----------------------------------------------------------
def getAllFilenames(filename, start_index=None, end_index=None): def getAllFilenames(filename, start_index=None, end_index=None):
...@@ -150,8 +155,8 @@ def getAllFilenames(filename, start_index=None, end_index=None): ...@@ -150,8 +155,8 @@ def getAllFilenames(filename, start_index=None, end_index=None):
Returns: Returns:
(data_filenames, indices_filenames, data_checksum_filenames, indices_checksum_filenames) (data_filenames, indices_filenames,
data_checksum_filenames, indices_checksum_filenames)
""" """
index_re = re.compile(r'^.*\.(\d+)\.(\d+)\.(data|index)(.checksum)?$') index_re = re.compile(r'^.*\.(\d+)\.(\d+)\.(data|index)(.checksum)?$')
...@@ -191,7 +196,7 @@ def getAllFilenames(filename, start_index=None, end_index=None): ...@@ -191,7 +196,7 @@ def getAllFilenames(filename, start_index=None, end_index=None):
return (data_filenames, indices_filenames, data_checksum_filenames, indices_checksum_filenames) return (data_filenames, indices_filenames, data_checksum_filenames, indices_checksum_filenames)
#---------------------------------------------------------- # ----------------------------------------------------------
class DataSource(object): class DataSource(object):
...@@ -273,7 +278,7 @@ class DataSource(object): ...@@ -273,7 +278,7 @@ class DataSource(object):
self.ready = True self.ready = True
#---------------------------------------------------------- # ----------------------------------------------------------
class CachedDataSource(DataSource): class CachedDataSource(DataSource):
...@@ -360,8 +365,8 @@ class CachedDataSource(DataSource): ...@@ -360,8 +365,8 @@ class CachedDataSource(DataSource):
3. Contiguous indices if they are present 3. Contiguous indices if they are present
""" """
# Make sure that we have a perfect match between data files and checksum # Make sure that we have a perfect match between data files and
# files # checksum files
checksum_filenames_noext = [os.path.splitext(f)[0] for f in checksum_filenames] checksum_filenames_noext = [os.path.splitext(f)[0] for f in checksum_filenames]
if data_filenames != checksum_filenames_noext: if data_filenames != checksum_filenames_noext:
...@@ -498,7 +503,7 @@ class CachedDataSource(DataSource): ...@@ -498,7 +503,7 @@ class CachedDataSource(DataSource):
return (data, infos.start_index, infos.end_index) return (data, infos.start_index, infos.end_index)
#---------------------------------------------------------- # ----------------------------------------------------------
class DatabaseOutputDataSource(DataSource): class DatabaseOutputDataSource(DataSource):
...@@ -617,7 +622,7 @@ class DatabaseOutputDataSource(DataSource): ...@@ -617,7 +622,7 @@ class DatabaseOutputDataSource(DataSource):
return (data, infos.start_index, infos.end_index) return (data, infos.start_index, infos.end_index)
#---------------------------------------------------------- # ----------------------------------------------------------
class RemoteDataSource(DataSource): class RemoteDataSource(DataSource):
...@@ -748,7 +753,7 @@ class RemoteDataSource(DataSource): ...@@ -748,7 +753,7 @@ class RemoteDataSource(DataSource):
self.ready = True self.ready = True
#---------------------------------------------------------- # ----------------------------------------------------------
class DataSink(object): class DataSink(object):
...@@ -777,14 +782,17 @@ class DataSink(object): ...@@ -777,14 +782,17 @@ class DataSink(object):
@abc.abstractmethod @abc.abstractmethod
def isConnected(self): def isConnected(self):
pass """Returns whether the data sink is connected"""
pass
def close(self): def close(self):
"""Closes the data sink"""
pass pass
#---------------------------------------------------------- # ----------------------------------------------------------
class StdoutDataSink(DataSink): class StdoutDataSink(DataSink):
...@@ -834,7 +842,7 @@ class StdoutDataSink(DataSink): ...@@ -834,7 +842,7 @@ class StdoutDataSink(DataSink):
return True return True
#---------------------------------------------------------- # ----------------------------------------------------------
class CachedDataSink(DataSink): class CachedDataSink(DataSink):
...@@ -867,12 +875,12 @@ class CachedDataSink(DataSink): ...@@ -867,12 +875,12 @@ class CachedDataSink(DataSink):
filename (str): Name of the file to generate filename (str): Name of the file to generate
dataformat (dataformat.DataFormat): The dataformat to be used dataformat (dataformat.DataFormat): The dataformat to be used
inside this file. All objects stored inside this file will respect that inside this file. All objects stored inside this file will respect
format. that format.
encoding (str): String defining the encoding to be used for encoding the encoding (str): String defining the encoding to be used for encoding
data. Only a few options are supported: ``binary`` (the default) or the data. Only a few options are supported: ``binary``
``json`` (debugging purposes). (the default) or ``json`` (debugging purposes).
""" """
...@@ -1016,7 +1024,7 @@ class CachedDataSink(DataSink): ...@@ -1016,7 +1024,7 @@ class CachedDataSink(DataSink):
return (self.filename is not None) return (self.filename is not None)
#---------------------------------------------------------- # ----------------------------------------------------------
def load_data_index(cache_root, hash_path): def load_data_index(cache_root, hash_path):
...@@ -1072,7 +1080,7 @@ def load_data_index(cache_root, hash_path): ...@@ -1072,7 +1080,7 @@ def load_data_index(cache_root, hash_path):
return sorted(retval) + [end_index + 1] return sorted(retval) + [end_index + 1]
#---------------------------------------------------------- # ----------------------------------------------------------
def load_data_index_db(cache_root, hash_path): def load_data_index_db(cache_root, hash_path):
...@@ -1105,7 +1113,7 @@ def load_data_index_db(cache_root, hash_path): ...@@ -1105,7 +1113,7 @@ def load_data_index_db(cache_root, hash_path):
return retval return retval
#---------------------------------------------------------- # ----------------------------------------------------------
def _foundCommonIndices(lst): def _foundCommonIndices(lst):
...@@ -1119,7 +1127,7 @@ def _foundCommonIndices(lst): ...@@ -1119,7 +1127,7 @@ def _foundCommonIndices(lst):
return common_indices return common_indices
#---------------------------------------------------------- # ----------------------------------------------------------
def foundSplitRanges(lst, n_split): def foundSplitRanges(lst, n_split):
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment