Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
bob
bob.pad.base
Commits
0ee60e58
Commit
0ee60e58
authored
Dec 04, 2020
by
Amir MOHAMMADI
Browse files
[vanilla-pad] Improve the dask client option
parent
da4bdcd5
Pipeline
#46422
passed with stage
in 8 minutes and 18 seconds
Changes
1
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
bob/pad/base/script/vanilla_pad.py
View file @
0ee60e58
"""Executes PAD pipeline"""
from
bob.bio.base.script.vanilla_biometrics
import
VALID_DASK_CLIENT_STRINGS
import
click
from
bob.extension.scripts.click_helper
import
ConfigCommand
from
bob.extension.scripts.click_helper
import
ResourceOption
...
...
@@ -37,9 +38,11 @@ from bob.extension.scripts.click_helper import verbosity_option
@
click
.
option
(
"--dask-client"
,
"-l"
,
required
=
False
,
cls
=
ResourceOption
,
entry_point_group
=
"dask.client"
,
string_exceptions
=
VALID_DASK_CLIENT_STRINGS
,
default
=
"single-threaded"
,
help
=
"Dask client for the execution of the pipeline."
,
cls
=
ResourceOption
,
)
@
click
.
option
(
"--group"
,
...
...
@@ -78,6 +81,7 @@ def vanilla_pad(ctx, pipeline, database, dask_client, groups, output, checkpoint
import
bob.pipelines
as
mario
import
dask.bag
from
bob.extension.scripts.click_helper
import
log_parameters
from
bob.pipelines.distributed.sge
import
get_resource_requirements
logger
=
logging
.
getLogger
(
__name__
)
log_parameters
(
logger
)
...
...
@@ -120,7 +124,8 @@ def vanilla_pad(ctx, pipeline, database, dask_client, groups, output, checkpoint
pattern
=
f
"
{
prefix
}
*
{
postfix
}
"
os
.
makedirs
(
os
.
path
.
dirname
(
prefix
),
exist_ok
=
True
)
logger
.
info
(
"Writing bag results into files ..."
)
result
.
to_textfiles
(
pattern
,
last_endline
=
True
,
scheduler
=
dask_client
)
resources
=
get_resource_requirements
(
pipeline
)
result
.
to_textfiles
(
pattern
,
last_endline
=
True
,
scheduler
=
dask_client
,
resources
=
resources
)
with
open
(
scores_path
,
"w"
)
as
f
:
# concatenate scores into one score file
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment