Commit 3aef698b authored by Amir MOHAMMADI's avatar Amir MOHAMMADI

Ran black and docformatter

parent e6330253
Pipeline #39311 passed with stage
in 10 minutes and 45 seconds
...@@ -21,15 +21,13 @@ from .sge_queues import QUEUE_DEFAULT ...@@ -21,15 +21,13 @@ from .sge_queues import QUEUE_DEFAULT
class SGEIdiapJob(Job): class SGEIdiapJob(Job):
""" """Launches a SGE Job in the IDIAP cluster. This class basically encodes
Launches a SGE Job in the IDIAP cluster. the CLI command that bootstrap the worker in a SGE job. Check here
This class basically encodes the CLI command that bootstrap the worker `https://distributed.dask.org/en/latest/resources.html#worker-resources`
in a SGE job. Check here `https://distributed.dask.org/en/latest/resources.html#worker-resources` for more information for more information.
..note: This is class is temporary. It's basically a copy from SGEJob from dask_jobqueue. ..note: This is class is temporary. It's basically a copy from SGEJob from dask_jobqueue.
The difference is that here I'm also handling the dask job resources tag (which is not handled anywhere). This has to be patched in the Job class. Please follow here `https://github.com/dask/dask-jobqueue/issues/378` to get news about this patch The difference is that here I'm also handling the dask job resources tag (which is not handled anywhere). This has to be patched in the Job class. Please follow here `https://github.com/dask/dask-jobqueue/issues/378` to get news about this patch
""" """
submit_command = "qsub" submit_command = "qsub"
...@@ -101,9 +99,7 @@ class SGEIdiapJob(Job): ...@@ -101,9 +99,7 @@ class SGEIdiapJob(Job):
def get_max_jobs(queue_dict): def get_max_jobs(queue_dict):
""" """Given a queue list, get the max number of possible jobs."""
Given a queue list, get the max number of possible jobs
"""
return max( return max(
[queue_dict[r]["max_jobs"] for r in queue_dict if "max_jobs" in queue_dict[r]] [queue_dict[r]["max_jobs"] for r in queue_dict if "max_jobs" in queue_dict[r]]
...@@ -111,7 +107,8 @@ def get_max_jobs(queue_dict): ...@@ -111,7 +107,8 @@ def get_max_jobs(queue_dict):
class SGEMultipleQueuesCluster(JobQueueCluster): class SGEMultipleQueuesCluster(JobQueueCluster):
""" Launch Dask jobs in the SGE cluster allowing the request of multiple queus """Launch Dask jobs in the SGE cluster allowing the request of multiple
queus.
Parameters Parameters
---------- ----------
...@@ -199,8 +196,6 @@ class SGEMultipleQueuesCluster(JobQueueCluster): ...@@ -199,8 +196,6 @@ class SGEMultipleQueuesCluster(JobQueueCluster):
>>> cluster = SGEIdiapCluster(sge_job_spec=Q_1DAY_GPU_SPEC) # doctest: +SKIP >>> cluster = SGEIdiapCluster(sge_job_spec=Q_1DAY_GPU_SPEC) # doctest: +SKIP
>>> cluster.adapt(Adaptive=AdaptiveIdiap,minimum=2, maximum=10) # doctest: +SKIP >>> cluster.adapt(Adaptive=AdaptiveIdiap,minimum=2, maximum=10) # doctest: +SKIP
>>> client = Client(cluster) # doctest: +SKIP >>> client = Client(cluster) # doctest: +SKIP
""" """
def __init__( def __init__(
...@@ -270,10 +265,7 @@ class SGEMultipleQueuesCluster(JobQueueCluster): ...@@ -270,10 +265,7 @@ class SGEMultipleQueuesCluster(JobQueueCluster):
self.adapt(minimum=min_jobs, maximum=max_jobs, wait_count=60, interval=1000) self.adapt(minimum=min_jobs, maximum=max_jobs, wait_count=60, interval=1000)
def _get_worker_spec_options(self, job_spec): def _get_worker_spec_options(self, job_spec):
""" """Craft a dask worker_spec to be used in the qsub command."""
Craft a dask worker_spec to be used in the qsub command
"""
def _get_key_from_spec(spec, key): def _get_key_from_spec(spec, key):
return spec[key] if key in spec else "" return spec[key] if key in spec else ""
...@@ -307,8 +299,7 @@ class SGEMultipleQueuesCluster(JobQueueCluster): ...@@ -307,8 +299,7 @@ class SGEMultipleQueuesCluster(JobQueueCluster):
} }
def scale(self, n_jobs, sge_job_spec_key="default"): def scale(self, n_jobs, sge_job_spec_key="default"):
""" """Launch an SGE job in the Idiap SGE cluster.
Launch an SGE job in the Idiap SGE cluster
Parameters Parameters
---------- ----------
...@@ -317,7 +308,7 @@ class SGEMultipleQueuesCluster(JobQueueCluster): ...@@ -317,7 +308,7 @@ class SGEMultipleQueuesCluster(JobQueueCluster):
Quantity of jobs to scale Quantity of jobs to scale
sge_job_spec_key: str sge_job_spec_key: str
One of the specs `SGEIdiapCluster.sge_job_spec` One of the specs `SGEIdiapCluster.sge_job_spec`
""" """
if n_jobs == 0: if n_jobs == 0:
...@@ -335,14 +326,18 @@ class SGEMultipleQueuesCluster(JobQueueCluster): ...@@ -335,14 +326,18 @@ class SGEMultipleQueuesCluster(JobQueueCluster):
return super(JobQueueCluster, self).scale(n_jobs, memory=None, cores=n_cores) return super(JobQueueCluster, self).scale(n_jobs, memory=None, cores=n_cores)
def scale_up(self, n_jobs, sge_job_spec_key=None): def scale_up(self, n_jobs, sge_job_spec_key=None):
""" """Scale cluster up.
Scale cluster up. This is supposed to be used by the scheduler while dynamically allocating resources
This is supposed to be used by the scheduler while dynamically
allocating resources
""" """
return self.scale(n_jobs, sge_job_spec_key) return self.scale(n_jobs, sge_job_spec_key)
async def scale_down(self, workers, sge_job_spec_key=None): async def scale_down(self, workers, sge_job_spec_key=None):
""" """Scale cluster down.
Scale cluster down. This is supposed to be used by the scheduler while dynamically allocating resources
This is supposed to be used by the scheduler while dynamically
allocating resources
""" """
await super().scale_down(workers) await super().scale_down(workers)
...@@ -351,9 +346,8 @@ class SGEMultipleQueuesCluster(JobQueueCluster): ...@@ -351,9 +346,8 @@ class SGEMultipleQueuesCluster(JobQueueCluster):
class AdaptiveMultipleQueue(Adaptive): class AdaptiveMultipleQueue(Adaptive):
""" """Custom mechanism to adaptively allocate workers based on scheduler load.
Custom mechanism to adaptively allocate workers based on scheduler load
This custom implementation extends the `Adaptive.recommendations` by looking This custom implementation extends the `Adaptive.recommendations` by looking
at the `distributed.scheduler.TaskState.resource_restrictions`. at the `distributed.scheduler.TaskState.resource_restrictions`.
...@@ -362,13 +356,11 @@ class AdaptiveMultipleQueue(Adaptive): ...@@ -362,13 +356,11 @@ class AdaptiveMultipleQueue(Adaptive):
.. note :: .. note ::
If a certain task has the status `no-worker` and it has resource_restrictions, the scheduler should If a certain task has the status `no-worker` and it has resource_restrictions, the scheduler should
request a job matching those resource restrictions request a job matching those resource restrictions
""" """
async def recommendations(self, target: int) -> dict: async def recommendations(self, target: int) -> dict:
""" """Make scale up/down recommendations based on current state and
Make scale up/down recommendations based on current state and target target."""
"""
plan = self.plan plan = self.plan
requested = self.requested requested = self.requested
...@@ -440,12 +432,11 @@ class AdaptiveMultipleQueue(Adaptive): ...@@ -440,12 +432,11 @@ class AdaptiveMultipleQueue(Adaptive):
class SchedulerResourceRestriction(Scheduler): class SchedulerResourceRestriction(Scheduler):
""" """Idiap extended distributed scheduler.
Idiap extended distributed scheduler
This scheduler extends `Scheduler` by just adding a handler This scheduler extends `Scheduler` by just adding a handler that
that fetches, at every scheduler cycle, the resource restrictions of fetches, at every scheduler cycle, the resource restrictions of a
a task that has status `no-worker` task that has status `no-worker`
""" """
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
...@@ -455,9 +446,8 @@ class SchedulerResourceRestriction(Scheduler): ...@@ -455,9 +446,8 @@ class SchedulerResourceRestriction(Scheduler):
] = self.get_no_worker_tasks_resource_restrictions ] = self.get_no_worker_tasks_resource_restrictions
def get_no_worker_tasks_resource_restrictions(self, comm=None): def get_no_worker_tasks_resource_restrictions(self, comm=None):
""" """Get the a task resource restrictions for jobs that has the status
Get the a task resource restrictions for jobs that has the status 'no-worker' 'no-worker'."""
"""
resource_restrictions = [] resource_restrictions = []
for k in self.tasks: for k in self.tasks:
......
...@@ -3,10 +3,10 @@ ...@@ -3,10 +3,10 @@
# Tiago de Freitas Pereira <tiago.pereira@idiap.ch> # Tiago de Freitas Pereira <tiago.pereira@idiap.ch>
""" """This queue setup has a DEMANDING arrangement.
This queue setup has a DEMANDING arrangement.
For CPU jobs, it prioritizes q_1day and io_big For CPU jobs, it prioritizes q_1day and io_big This HAS to be the
This HAS to be the default default
""" """
QUEUE_DEFAULT = { QUEUE_DEFAULT = {
"default": { "default": {
......
"""Base definition of sample""" """Base definition of sample."""
from collections.abc import MutableSequence from collections.abc import MutableSequence
def _copy_attributes(s, d): def _copy_attributes(s, d):
"""Copies attributes from a dictionary to self """Copies attributes from a dictionary to self."""
"""
s.__dict__.update( s.__dict__.update(
dict( dict(
(k, v) (k, v)
...@@ -25,7 +24,8 @@ class _ReprMixin: ...@@ -25,7 +24,8 @@ class _ReprMixin:
class Sample(_ReprMixin): class Sample(_ReprMixin):
"""Representation of sample. A Sample is a simple container that wraps a data-point (see :ref:`bob.pipelines.sample`) """Representation of sample. A Sample is a simple container that wraps a
data-point (see :ref:`bob.pipelines.sample`)
Each sample must have the following attributes: Each sample must have the following attributes:
...@@ -41,7 +41,6 @@ class Sample(_ReprMixin): ...@@ -41,7 +41,6 @@ class Sample(_ReprMixin):
parent : object parent : object
A parent object from which to inherit all other attributes (except A parent object from which to inherit all other attributes (except
``data``) ``data``)
""" """
def __init__(self, data, parent=None, **kwargs): def __init__(self, data, parent=None, **kwargs):
...@@ -52,7 +51,7 @@ class Sample(_ReprMixin): ...@@ -52,7 +51,7 @@ class Sample(_ReprMixin):
class DelayedSample(_ReprMixin): class DelayedSample(_ReprMixin):
"""Representation of sample that can be loaded via a callable """Representation of sample that can be loaded via a callable.
The optional ``**kwargs`` argument allows you to attach more attributes to The optional ``**kwargs`` argument allows you to attach more attributes to
this sample instance. this sample instance.
...@@ -72,7 +71,6 @@ class DelayedSample(_ReprMixin): ...@@ -72,7 +71,6 @@ class DelayedSample(_ReprMixin):
kwargs : dict kwargs : dict
Further attributes of this sample, to be stored and eventually Further attributes of this sample, to be stored and eventually
transmitted to transformed versions of the sample transmitted to transformed versions of the sample
""" """
def __init__(self, load, parent=None, **kwargs): def __init__(self, load, parent=None, **kwargs):
...@@ -84,7 +82,7 @@ class DelayedSample(_ReprMixin): ...@@ -84,7 +82,7 @@ class DelayedSample(_ReprMixin):
@property @property
def data(self): def data(self):
"""Loads the data from the disk file""" """Loads the data from the disk file."""
if self._data is None: if self._data is None:
self._data = self.load() self._data = self.load()
return self._data return self._data
...@@ -92,8 +90,8 @@ class DelayedSample(_ReprMixin): ...@@ -92,8 +90,8 @@ class DelayedSample(_ReprMixin):
class SampleSet(MutableSequence, _ReprMixin): class SampleSet(MutableSequence, _ReprMixin):
"""A set of samples with extra attributes """A set of samples with extra attributes
https://docs.python.org/3/library/collections.abc.html#collections-abstract-base-classes https://docs.python.org/3/library/collections.abc.html#collections-
""" abstract-base-classes."""
def __init__(self, samples, parent=None, **kwargs): def __init__(self, samples, parent=None, **kwargs):
self.samples = samples self.samples = samples
......
...@@ -18,7 +18,8 @@ def _offset_add_func(X, offset=1): ...@@ -18,7 +18,8 @@ def _offset_add_func(X, offset=1):
class DummyWithFit(TransformerMixin, BaseEstimator): class DummyWithFit(TransformerMixin, BaseEstimator):
"""See https://scikit-learn.org/stable/developers/develop.html and """See https://scikit-learn.org/stable/developers/develop.html and
https://github.com/scikit-learn-contrib/project-template/blob/master/skltemplate/_template.py""" https://github.com/scikit-learn-contrib/project-
template/blob/master/skltemplate/_template.py."""
def fit(self, X, y=None): def fit(self, X, y=None):
X = check_array(X) X = check_array(X)
...@@ -45,7 +46,8 @@ class DummyWithFit(TransformerMixin, BaseEstimator): ...@@ -45,7 +46,8 @@ class DummyWithFit(TransformerMixin, BaseEstimator):
class DummyTransformer(TransformerMixin, BaseEstimator): class DummyTransformer(TransformerMixin, BaseEstimator):
"""See https://scikit-learn.org/stable/developers/develop.html and """See https://scikit-learn.org/stable/developers/develop.html and
https://github.com/scikit-learn-contrib/project-template/blob/master/skltemplate/_template.py""" https://github.com/scikit-learn-contrib/project-
template/blob/master/skltemplate/_template.py."""
def __init__(self, picklable=True, i=None, **kwargs): def __init__(self, picklable=True, i=None, **kwargs):
super().__init__(**kwargs) super().__init__(**kwargs)
......
...@@ -4,17 +4,16 @@ from ..wrappers import wrap ...@@ -4,17 +4,16 @@ from ..wrappers import wrap
def SampleFunctionTransformer(**kwargs): def SampleFunctionTransformer(**kwargs):
"""Class that transforms Scikit learn FunctionTransformer """Class that transforms Scikit learn FunctionTransformer (https://scikit-l
(https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.FunctionTransformer.html) earn.org/stable/modules/generated/sklearn.preprocessing.FunctionTransformer
work with :any:`Sample`-based pipelines. .html) work with :any:`Sample`-based pipelines."""
"""
return wrap([FunctionTransformer, "sample"], **kwargs) return wrap([FunctionTransformer, "sample"], **kwargs)
def CheckpointSampleFunctionTransformer(**kwargs): def CheckpointSampleFunctionTransformer(**kwargs):
"""Class that transforms Scikit learn FunctionTransformer """Class that transforms Scikit learn FunctionTransformer (https://scikit-l
(https://scikit-learn.org/stable/modules/generated/sklearn.preprocessing.FunctionTransformer.html) earn.org/stable/modules/generated/sklearn.preprocessing.FunctionTransformer
work with :any:`Sample`-based pipelines. .html) work with :any:`Sample`-based pipelines.
Furthermore, it makes it checkpointable Furthermore, it makes it checkpointable
""" """
......
...@@ -9,8 +9,8 @@ def linearize(X): ...@@ -9,8 +9,8 @@ def linearize(X):
class Linearize(FunctionTransformer): class Linearize(FunctionTransformer):
"""Extracts features by simply concatenating all elements of the data into one long vector. """Extracts features by simply concatenating all elements of the data into
""" one long vector."""
def __init__(self, **kwargs): def __init__(self, **kwargs):
super().__init__(func=linearize, **kwargs) super().__init__(func=linearize, **kwargs)
......
...@@ -3,14 +3,11 @@ from ..wrappers import wrap ...@@ -3,14 +3,11 @@ from ..wrappers import wrap
def SamplePCA(**kwargs): def SamplePCA(**kwargs):
""" """Enables SAMPLE handling for :any:`sklearn.decomposition.PCA`"""
Enables SAMPLE handling for :any:`sklearn.decomposition.PCA`
"""
return wrap([PCA, "sample"], **kwargs) return wrap([PCA, "sample"], **kwargs)
def CheckpointSamplePCA(**kwargs): def CheckpointSamplePCA(**kwargs):
""" """Enables SAMPLE and CHECKPOINTIN handling for
Enables SAMPLE and CHECKPOINTIN handling for :any:`sklearn.decomposition.PCA` :any:`sklearn.decomposition.PCA`"""
"""
return wrap([PCA, "sample", "checkpoint"], **kwargs) return wrap([PCA, "sample", "checkpoint"], **kwargs)
...@@ -3,11 +3,8 @@ import nose ...@@ -3,11 +3,8 @@ import nose
import numpy as np import numpy as np
def is_picklable(obj): def is_picklable(obj):
""" """Test if an object is picklable or not."""
Test if an object is picklable or not
"""
try: try:
pickle.dumps(obj) pickle.dumps(obj)
except TypeError: except TypeError:
...@@ -19,9 +16,7 @@ def is_picklable(obj): ...@@ -19,9 +16,7 @@ def is_picklable(obj):
def assert_picklable(obj): def assert_picklable(obj):
""" """Test if an object is picklable or not."""
Test if an object is picklable or not
"""
string = pickle.dumps(obj) string = pickle.dumps(obj)
new_obj = pickle.loads(string) new_obj = pickle.loads(string)
...@@ -50,8 +45,8 @@ def is_estimator_stateless(estimator): ...@@ -50,8 +45,8 @@ def is_estimator_stateless(estimator):
def _generate_features(reader, paths, same_size=False): def _generate_features(reader, paths, same_size=False):
"""Load and stack features in a memory efficient way. This function is meant """Load and stack features in a memory efficient way. This function is
to be used inside :py:func:`vstack_features`. meant to be used inside :py:func:`vstack_features`.
Parameters Parameters
---------- ----------
......
"""Scikit-learn Estimator Wrappers""" """Scikit-learn Estimator Wrappers."""
from .sample import DelayedSample, SampleSet from .sample import DelayedSample, SampleSet
from .utils import is_estimator_stateless, samples_to_np_array from .utils import is_estimator_stateless, samples_to_np_array
import dask.bag import dask.bag
...@@ -28,7 +28,7 @@ def _frmt(estimator, limit=40): ...@@ -28,7 +28,7 @@ def _frmt(estimator, limit=40):
class BaseWrapper(BaseEstimator): class BaseWrapper(BaseEstimator):
"""The base class for all wrappers""" """The base class for all wrappers."""
def _more_tags(self): def _more_tags(self):
return self.estimator._more_tags() return self.estimator._more_tags()
...@@ -66,7 +66,8 @@ class DelayedSamplesCall: ...@@ -66,7 +66,8 @@ class DelayedSamplesCall:
class SampleWrapper(BaseWrapper, TransformerMixin): class SampleWrapper(BaseWrapper, TransformerMixin):
"""Wraps scikit-learn estimators to work with :any:`Sample`-based pipelines. """Wraps scikit-learn estimators to work with :any:`Sample`-based
pipelines.
Do not use this class except for scikit-learn estimators. Do not use this class except for scikit-learn estimators.
...@@ -164,7 +165,8 @@ class SampleWrapper(BaseWrapper, TransformerMixin): ...@@ -164,7 +165,8 @@ class SampleWrapper(BaseWrapper, TransformerMixin):
class CheckpointWrapper(BaseWrapper, TransformerMixin): class CheckpointWrapper(BaseWrapper, TransformerMixin):
"""Wraps :any:`Sample`-based estimators so the results are saved in disk.""" """Wraps :any:`Sample`-based estimators so the results are saved in
disk."""
def __init__( def __init__(
self, self,
...@@ -303,7 +305,7 @@ class CheckpointWrapper(BaseWrapper, TransformerMixin): ...@@ -303,7 +305,7 @@ class CheckpointWrapper(BaseWrapper, TransformerMixin):
class DaskWrapper(BaseWrapper, TransformerMixin): class DaskWrapper(BaseWrapper, TransformerMixin):
"""Wraps Scikit estimators to handle Dask Bags as input """Wraps Scikit estimators to handle Dask Bags as input.
Parameters Parameters
---------- ----------
...@@ -319,7 +321,6 @@ class DaskWrapper(BaseWrapper, TransformerMixin): ...@@ -319,7 +321,6 @@ class DaskWrapper(BaseWrapper, TransformerMixin):
a future delayed(self.transform).compute(resources=resource_tape) so a future delayed(self.transform).compute(resources=resource_tape) so
dask scheduler can place this task in a particular resource dask scheduler can place this task in a particular resource
(e.g GPU) (e.g GPU)
""" """
def __init__( def __init__(
...@@ -414,7 +415,6 @@ class ToDaskBag(TransformerMixin, BaseEstimator): ...@@ -414,7 +415,6 @@ class ToDaskBag(TransformerMixin, BaseEstimator):
def wrap(bases, estimator=None, **kwargs): def wrap(bases, estimator=None, **kwargs):
"""Wraps several estimators inside each other. """Wraps several estimators inside each other.
Parameters Parameters
---------- ----------
bases : list bases : list
...@@ -480,9 +480,7 @@ def wrap(bases, estimator=None, **kwargs): ...@@ -480,9 +480,7 @@ def wrap(bases, estimator=None, **kwargs):
def dask_tags(estimator): def dask_tags(estimator):
""" """Recursively collects resource_tags in dasked estimators."""
Recursively collects resource_tags in dasked estimators
"""
tags = {} tags = {}
if hasattr(estimator, "estimator"): if hasattr(estimator, "estimator"):
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment