Skip to content
Snippets Groups Projects
Commit a201b15e authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

[sphinx] Updated examples

parent e1b35e5b
Branches
Tags
1 merge request!51[dask][sge] Multiqueue updates
Pipeline #46064 passed
...@@ -25,7 +25,7 @@ class MyFitTranformer(TransformerMixin, BaseEstimator): ...@@ -25,7 +25,7 @@ class MyFitTranformer(TransformerMixin, BaseEstimator):
def __init__(self): def __init__(self):
self._fit_model = None self._fit_model = None
def transform(self, X): def transform(self, X, metadata=None):
# Transform `X` # Transform `X`
return [x @ self._fit_model for x in X] return [x @ self._fit_model for x in X]
...@@ -40,7 +40,7 @@ X = numpy.zeros((2, 2)) ...@@ -40,7 +40,7 @@ X = numpy.zeros((2, 2))
X_as_sample = [Sample(X, key=str(i), metadata=1) for i in range(10)] X_as_sample = [Sample(X, key=str(i), metadata=1) for i in range(10)]
# Building an arbitrary pipeline # Building an arbitrary pipeline
model_path = "~/dask_tmp" model_path = "./dask_tmp"
os.makedirs(model_path, exist_ok=True) os.makedirs(model_path, exist_ok=True)
pipeline = make_pipeline(MyTransformer(), MyFitTranformer()) pipeline = make_pipeline(MyTransformer(), MyFitTranformer())
...@@ -48,13 +48,15 @@ pipeline = make_pipeline(MyTransformer(), MyFitTranformer()) ...@@ -48,13 +48,15 @@ pipeline = make_pipeline(MyTransformer(), MyFitTranformer())
pipeline = bob.pipelines.wrap( pipeline = bob.pipelines.wrap(
["sample", "checkpoint", "dask"], ["sample", "checkpoint", "dask"],
pipeline, pipeline,
model_path=model_path, model_path=os.path.join(model_path, "model.pickle"),
features_dir=model_path,
transform_extra_arguments=(("metadata", "metadata"),), transform_extra_arguments=(("metadata", "metadata"),),
) )
# Create a dask graph from a pipeline # Create a dask graph from a pipeline
# Run the task graph in the local computer in a single tread # Run the task graph in the local computer in a single tread
X_transformed = pipeline.fit_transform(X_as_sample).compute(scheduler="single-threaded") X_transformed = pipeline.fit_transform(X_as_sample).compute(scheduler="single-threaded")
import shutil import shutil
shutil.rmtree(model_path) shutil.rmtree(model_path)
...@@ -28,7 +28,7 @@ class MyFitTranformer(TransformerMixin, BaseEstimator): ...@@ -28,7 +28,7 @@ class MyFitTranformer(TransformerMixin, BaseEstimator):
def __init__(self): def __init__(self):
self._fit_model = None self._fit_model = None
def transform(self, X): def transform(self, X, metadata=None):
# Transform `X` # Transform `X`
return [x @ self._fit_model for x in X] return [x @ self._fit_model for x in X]
...@@ -43,7 +43,7 @@ X = numpy.zeros((2, 2)) ...@@ -43,7 +43,7 @@ X = numpy.zeros((2, 2))
X_as_sample = [Sample(X, key=str(i), metadata=1) for i in range(10)] X_as_sample = [Sample(X, key=str(i), metadata=1) for i in range(10)]
# Building an arbitrary pipeline # Building an arbitrary pipeline
model_path = "~/dask_tmp" model_path = "./dask_tmp"
os.makedirs(model_path, exist_ok=True) os.makedirs(model_path, exist_ok=True)
pipeline = make_pipeline(MyTransformer(), MyFitTranformer()) pipeline = make_pipeline(MyTransformer(), MyFitTranformer())
...@@ -51,7 +51,8 @@ pipeline = make_pipeline(MyTransformer(), MyFitTranformer()) ...@@ -51,7 +51,8 @@ pipeline = make_pipeline(MyTransformer(), MyFitTranformer())
pipeline = bob.pipelines.wrap( pipeline = bob.pipelines.wrap(
["sample", "checkpoint", "dask"], ["sample", "checkpoint", "dask"],
pipeline, pipeline,
model_path=model_path, model_path=os.path.join(model_path, "model.pickle"),
features_dir=model_path,
transform_extra_arguments=(("metadata", "metadata"),), transform_extra_arguments=(("metadata", "metadata"),),
) )
...@@ -62,6 +63,7 @@ client = Client(cluster) # Creating the scheduler ...@@ -62,6 +63,7 @@ client = Client(cluster) # Creating the scheduler
# Run the task graph in the local computer in a single tread # Run the task graph in the local computer in a single tread
X_transformed = pipeline.fit_transform(X_as_sample).compute(scheduler=client) X_transformed = pipeline.fit_transform(X_as_sample).compute(scheduler=client)
import shutil import shutil
shutil.rmtree(model_path) shutil.rmtree(model_path)
...@@ -28,7 +28,7 @@ class MyFitTranformer(TransformerMixin, BaseEstimator): ...@@ -28,7 +28,7 @@ class MyFitTranformer(TransformerMixin, BaseEstimator):
def __init__(self): def __init__(self):
self._fit_model = None self._fit_model = None
def transform(self, X): def transform(self, X, metadata=None):
# Transform `X` # Transform `X`
return [x @ self._fit_model for x in X] return [x @ self._fit_model for x in X]
...@@ -43,7 +43,7 @@ X = numpy.zeros((2, 2)) ...@@ -43,7 +43,7 @@ X = numpy.zeros((2, 2))
X_as_sample = [Sample(X, key=str(i), metadata=1) for i in range(10)] X_as_sample = [Sample(X, key=str(i), metadata=1) for i in range(10)]
# Building an arbitrary pipeline # Building an arbitrary pipeline
model_path = "~/dask_tmp" model_path = "./dask_tmp"
os.makedirs(model_path, exist_ok=True) os.makedirs(model_path, exist_ok=True)
pipeline = make_pipeline(MyTransformer(), MyFitTranformer()) pipeline = make_pipeline(MyTransformer(), MyFitTranformer())
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment