Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
bob.pipelines
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
4
Issues
4
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
bob
bob.pipelines
Commits
b254c180
Commit
b254c180
authored
Nov 24, 2020
by
Tiago de Freitas Pereira
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
[dask] Simplified the way multiqueue is set
parent
22b36d8c
Changes
3
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
57 additions
and
37 deletions
+57
-37
bob/pipelines/distributed/sge.py
bob/pipelines/distributed/sge.py
+29
-7
bob/pipelines/distributed/sge_queues.py
bob/pipelines/distributed/sge_queues.py
+15
-17
bob/pipelines/wrappers.py
bob/pipelines/wrappers.py
+13
-13
No files found.
bob/pipelines/distributed/sge.py
View file @
b254c180
...
...
@@ -145,9 +145,9 @@ class SGEMultipleQueuesCluster(JobQueueCluster):
Below follow a vanilla-example that will create a set of jobs on all.q:
>>> from bob.pipelines.distributed.sge import SGE
Idiap
Cluster # doctest: +SKIP
>>> from bob.pipelines.distributed.sge import SGE
MultipleQueues
Cluster # doctest: +SKIP
>>> from dask.distributed import Client # doctest: +SKIP
>>> cluster = SGE
Idiap
Cluster() # doctest: +SKIP
>>> cluster = SGE
MultipleQueues
Cluster() # doctest: +SKIP
>>> cluster.scale_up(10) # doctest: +SKIP
>>> client = Client(cluster) # doctest: +SKIP
...
...
@@ -162,7 +162,7 @@ class SGEMultipleQueuesCluster(JobQueueCluster):
... "resources": "",
... }
... }
>>> cluster = SGE
Idiap
Cluster(sge_job_spec=Q_1DAY_IO_BIG_SPEC) # doctest: +SKIP
>>> cluster = SGE
MultipleQueues
Cluster(sge_job_spec=Q_1DAY_IO_BIG_SPEC) # doctest: +SKIP
>>> cluster.scale_up(10) # doctest: +SKIP
>>> client = Client(cluster) # doctest: +SKIP
...
...
@@ -186,7 +186,7 @@ class SGEMultipleQueuesCluster(JobQueueCluster):
... "resources": {"GPU":1},
... },
... }
>>> cluster = SGE
Idiap
Cluster(sge_job_spec=Q_1DAY_GPU_SPEC) # doctest: +SKIP
>>> cluster = SGE
MultipleQueues
Cluster(sge_job_spec=Q_1DAY_GPU_SPEC) # doctest: +SKIP
>>> cluster.scale_up(10) # doctest: +SKIP
>>> cluster.scale_up(1, sge_job_spec_key="gpu") # doctest: +SKIP
>>> client = Client(cluster) # doctest: +SKIP
...
...
@@ -194,7 +194,7 @@ class SGEMultipleQueuesCluster(JobQueueCluster):
Adaptive job allocation can also be used via `AdaptiveIdiap` extension:
>>> cluster = SGE
Idiap
Cluster(sge_job_spec=Q_1DAY_GPU_SPEC) # doctest: +SKIP
>>> cluster = SGE
MultipleQueues
Cluster(sge_job_spec=Q_1DAY_GPU_SPEC) # doctest: +SKIP
>>> cluster.adapt(Adaptive=AdaptiveIdiap,minimum=2, maximum=10) # doctest: +SKIP
>>> client = Client(cluster) # doctest: +SKIP
"""
...
...
@@ -267,6 +267,28 @@ class SGEMultipleQueuesCluster(JobQueueCluster):
self
.
adapt
(
minimum
=
min_jobs
,
maximum
=
max_jobs
,
wait_count
=
5
,
interval
=
120
)
def
get_sge_resources
(
self
):
"""
Get the available resources from `SGEMultipleQueuesCluster.sge_job_spec`.
This is useful when it's necessary to set the resource available for
`.compute` method.
Check https://distributed.dask.org/en/latest/resources.html#resources-with-collections for more information
Example
-------
>>> cluster = SGEMultipleQueuesCluster(sge_job_spec=Q_1DAY_GPU_SPEC) # doctest: +SKIP
>>> client = Client(cluster) # doctest: +SKIP
>>> my_delayed_task.compute(scheduler=client, resources=resources) # doctest: +SKIP
"""
resources
=
[
list
(
self
.
sge_job_spec
[
k
][
"resources"
].
items
())[
0
]
for
k
in
self
.
sge_job_spec
if
self
.
sge_job_spec
[
k
][
"resources"
]
!=
""
]
return
dict
(
resources
)
def
_get_worker_spec_options
(
self
,
job_spec
):
"""Craft a dask worker_spec to be used in the qsub command."""
...
...
@@ -315,7 +337,7 @@ class SGEMultipleQueuesCluster(JobQueueCluster):
Quantity of jobs to scale
sge_job_spec_key: str
One of the specs `SGE
Idiap
Cluster.sge_job_spec`
One of the specs `SGE
MultipleQueues
Cluster.sge_job_spec`
"""
if
n_jobs
==
0
:
...
...
@@ -452,7 +474,7 @@ class SchedulerResourceRestriction(Scheduler):
allowed_failures
=
100
if
rc
.
get
(
"bob.pipelines.sge.allowed_failures"
)
is
None
else
rc
.
get
(
"bob.pipelines.sge.allowed_failures"
),
synchronize_worker_interval
=
"
6
0s"
,
synchronize_worker_interval
=
"
2
0s"
,
*
args
,
**
kwargs
,
)
...
...
bob/pipelines/distributed/sge_queues.py
View file @
b254c180
...
...
@@ -20,13 +20,12 @@ QUEUE_DEFAULT = {
"max_jobs"
:
24
,
"resources"
:
{
"q_1week"
:
1
},
},
"q_
short
_gpu"
:
{
"queue"
:
"q_
short
_gpu"
,
"q_
long
_gpu"
:
{
"queue"
:
"q_
long
_gpu"
,
"memory"
:
"30GB"
,
"io_big"
:
False
,
"resource_spec"
:
""
,
"max_jobs"
:
45
,
"resources"
:
{
"q_short_gpu"
:
1
},
"resources"
:
{
"q_long_gpu"
:
1
},
},
"q_gpu"
:
{
"queue"
:
"q_gpu"
,
...
...
@@ -35,20 +34,15 @@ QUEUE_DEFAULT = {
"resource_spec"
:
""
,
"resources"
:
{
"q_gpu"
:
1
},
},
"q_
long
_gpu"
:
{
"queue"
:
"q_
long
_gpu"
,
"q_
short
_gpu"
:
{
"queue"
:
"q_
short
_gpu"
,
"memory"
:
"30GB"
,
"io_big"
:
False
,
"resource_spec"
:
""
,
"resources"
:
{
"q_long_gpu"
:
1
},
"max_jobs"
:
45
,
"resources"
:
{
"q_short_gpu"
:
1
},
},
}
"""This queue setup has a DEMANDING arrangement.
For CPU jobs, it prioritizes q_1day and io_big This HAS to be the
default
"""
QUEUE_GPU
=
{
"default"
:
{
...
...
@@ -57,7 +51,7 @@ QUEUE_GPU = {
"io_big"
:
False
,
"resource_spec"
:
""
,
"max_jobs"
:
45
,
"resources"
:
{
"q_short_gpu"
:
1
}
,
"resources"
:
""
,
},
"q_1day"
:
{
"queue"
:
"q_1day"
,
...
...
@@ -82,6 +76,13 @@ QUEUE_GPU = {
"max_jobs"
:
45
,
"resources"
:
{
"q_short_gpu"
:
1
},
},
"q_long_gpu"
:
{
"queue"
:
"q_long_gpu"
,
"memory"
:
"30GB"
,
"io_big"
:
False
,
"resource_spec"
:
""
,
"resources"
:
{
"q_long_gpu"
:
1
},
},
"q_gpu"
:
{
"queue"
:
"q_gpu"
,
"memory"
:
"30GB"
,
...
...
@@ -97,6 +98,3 @@ QUEUE_GPU = {
"resources"
:
{
"q_long_gpu"
:
1
},
},
}
"""
This queue setup uses the q_short_gpu queue of the SGE.
"""
bob/pipelines/wrappers.py
View file @
b254c180
...
...
@@ -138,8 +138,7 @@ class SampleWrapper(BaseWrapper, TransformerMixin):
if
isinstance
(
samples
[
0
],
SampleSet
):
return
[
SampleSet
(
self
.
_samples_transform
(
sset
.
samples
,
method_name
),
parent
=
sset
,
self
.
_samples_transform
(
sset
.
samples
,
method_name
),
parent
=
sset
,
)
for
sset
in
samples
]
...
...
@@ -408,11 +407,7 @@ class DaskWrapper(BaseWrapper, TransformerMixin):
"""
def
__init__
(
self
,
estimator
,
fit_tag
=
None
,
transform_tag
=
None
,
**
kwargs
,
self
,
estimator
,
fit_tag
=
None
,
transform_tag
=
None
,
**
kwargs
,
):
super
().
__init__
(
**
kwargs
)
self
.
estimator
=
estimator
...
...
@@ -421,6 +416,9 @@ class DaskWrapper(BaseWrapper, TransformerMixin):
self
.
fit_tag
=
fit_tag
self
.
transform_tag
=
transform_tag
def
_make_dask_resource_tag
(
self
,
tag
):
return
[(
1
,
tag
)]
def
_dask_transform
(
self
,
X
,
method_name
):
graph_name
=
f"
{
_frmt
(
self
)
}
.
{
method_name
}
"
logger
.
debug
(
graph_name
)
...
...
@@ -432,7 +430,9 @@ class DaskWrapper(BaseWrapper, TransformerMixin):
_transf
.
__name__
=
graph_name
map_partitions
=
X
.
map_partitions
(
_transf
,
self
.
_dask_state
)
if
self
.
transform_tag
is
not
None
:
self
.
resource_tags
[
map_partitions
]
=
self
.
transform_tag
self
.
resource_tags
[
map_partitions
]
=
self
.
_make_dask_resource_tag
(
self
.
transform_tag
)
return
map_partitions
...
...
@@ -464,12 +464,12 @@ class DaskWrapper(BaseWrapper, TransformerMixin):
# change the name to have a better name in dask graphs
_fit
.
__name__
=
f"
{
_frmt
(
self
)
}
.fit"
self
.
_dask_state
=
delayed
(
_fit
)(
X
,
y
,
)
self
.
_dask_state
=
delayed
(
_fit
)(
X
,
y
,)
if
self
.
fit_tag
is
not
None
:
self
.
resource_tags
[
self
.
_dask_state
]
=
self
.
fit_tag
self
.
resource_tags
[
self
.
_dask_state
]
=
self
.
_make_dask_resource_tag
(
self
.
fit_tag
)
return
self
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment