Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
bob
bob.pipelines
Commits
a201b15e
Commit
a201b15e
authored
Nov 25, 2020
by
Tiago de Freitas Pereira
Browse files
[sphinx] Updated examples
parent
e1b35e5b
Pipeline
#46064
passed with stage
in 3 minutes and 41 seconds
Changes
3
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
doc/python/pipeline_example_dask.py
View file @
a201b15e
...
...
@@ -25,7 +25,7 @@ class MyFitTranformer(TransformerMixin, BaseEstimator):
def
__init__
(
self
):
self
.
_fit_model
=
None
def
transform
(
self
,
X
):
def
transform
(
self
,
X
,
metadata
=
None
):
# Transform `X`
return
[
x
@
self
.
_fit_model
for
x
in
X
]
...
...
@@ -40,7 +40,7 @@ X = numpy.zeros((2, 2))
X_as_sample
=
[
Sample
(
X
,
key
=
str
(
i
),
metadata
=
1
)
for
i
in
range
(
10
)]
# Building an arbitrary pipeline
model_path
=
"
~
/dask_tmp"
model_path
=
"
.
/dask_tmp"
os
.
makedirs
(
model_path
,
exist_ok
=
True
)
pipeline
=
make_pipeline
(
MyTransformer
(),
MyFitTranformer
())
...
...
@@ -48,13 +48,15 @@ pipeline = make_pipeline(MyTransformer(), MyFitTranformer())
pipeline
=
bob
.
pipelines
.
wrap
(
[
"sample"
,
"checkpoint"
,
"dask"
],
pipeline
,
model_path
=
model_path
,
model_path
=
os
.
path
.
join
(
model_path
,
"model.pickle"
),
features_dir
=
model_path
,
transform_extra_arguments
=
((
"metadata"
,
"metadata"
),),
)
# Create a dask graph from a pipeline
# Run the task graph in the local computer in a single tread
X_transformed
=
pipeline
.
fit_transform
(
X_as_sample
).
compute
(
scheduler
=
"single-threaded"
)
import
shutil
shutil
.
rmtree
(
model_path
)
doc/python/pipeline_example_dask_sge.py
View file @
a201b15e
...
...
@@ -28,7 +28,7 @@ class MyFitTranformer(TransformerMixin, BaseEstimator):
def
__init__
(
self
):
self
.
_fit_model
=
None
def
transform
(
self
,
X
):
def
transform
(
self
,
X
,
metadata
=
None
):
# Transform `X`
return
[
x
@
self
.
_fit_model
for
x
in
X
]
...
...
@@ -43,7 +43,7 @@ X = numpy.zeros((2, 2))
X_as_sample
=
[
Sample
(
X
,
key
=
str
(
i
),
metadata
=
1
)
for
i
in
range
(
10
)]
# Building an arbitrary pipeline
model_path
=
"
~
/dask_tmp"
model_path
=
"
.
/dask_tmp"
os
.
makedirs
(
model_path
,
exist_ok
=
True
)
pipeline
=
make_pipeline
(
MyTransformer
(),
MyFitTranformer
())
...
...
@@ -51,7 +51,8 @@ pipeline = make_pipeline(MyTransformer(), MyFitTranformer())
pipeline
=
bob
.
pipelines
.
wrap
(
[
"sample"
,
"checkpoint"
,
"dask"
],
pipeline
,
model_path
=
model_path
,
model_path
=
os
.
path
.
join
(
model_path
,
"model.pickle"
),
features_dir
=
model_path
,
transform_extra_arguments
=
((
"metadata"
,
"metadata"
),),
)
...
...
@@ -62,6 +63,7 @@ client = Client(cluster) # Creating the scheduler
# Run the task graph in the local computer in a single tread
X_transformed
=
pipeline
.
fit_transform
(
X_as_sample
).
compute
(
scheduler
=
client
)
import
shutil
shutil
.
rmtree
(
model_path
)
doc/python/pipeline_example_dask_sge_adaptive.py
View file @
a201b15e
...
...
@@ -28,7 +28,7 @@ class MyFitTranformer(TransformerMixin, BaseEstimator):
def
__init__
(
self
):
self
.
_fit_model
=
None
def
transform
(
self
,
X
):
def
transform
(
self
,
X
,
metadata
=
None
):
# Transform `X`
return
[
x
@
self
.
_fit_model
for
x
in
X
]
...
...
@@ -43,7 +43,7 @@ X = numpy.zeros((2, 2))
X_as_sample
=
[
Sample
(
X
,
key
=
str
(
i
),
metadata
=
1
)
for
i
in
range
(
10
)]
# Building an arbitrary pipeline
model_path
=
"
~
/dask_tmp"
model_path
=
"
.
/dask_tmp"
os
.
makedirs
(
model_path
,
exist_ok
=
True
)
pipeline
=
make_pipeline
(
MyTransformer
(),
MyFitTranformer
())
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment