From a201b15e8e874d7877ce15b3866ba73880b5a8cc Mon Sep 17 00:00:00 2001 From: Tiago Freitas Pereira <tiagofrepereira@gmail.com> Date: Wed, 25 Nov 2020 15:27:09 +0100 Subject: [PATCH] [sphinx] Updated examples --- doc/python/pipeline_example_dask.py | 8 +++++--- doc/python/pipeline_example_dask_sge.py | 8 +++++--- doc/python/pipeline_example_dask_sge_adaptive.py | 4 ++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/doc/python/pipeline_example_dask.py b/doc/python/pipeline_example_dask.py index e1df7d8..d6e365a 100644 --- a/doc/python/pipeline_example_dask.py +++ b/doc/python/pipeline_example_dask.py @@ -25,7 +25,7 @@ class MyFitTranformer(TransformerMixin, BaseEstimator): def __init__(self): self._fit_model = None - def transform(self, X): + def transform(self, X, metadata=None): # Transform `X` return [x @ self._fit_model for x in X] @@ -40,7 +40,7 @@ X = numpy.zeros((2, 2)) X_as_sample = [Sample(X, key=str(i), metadata=1) for i in range(10)] # Building an arbitrary pipeline -model_path = "~/dask_tmp" +model_path = "./dask_tmp" os.makedirs(model_path, exist_ok=True) pipeline = make_pipeline(MyTransformer(), MyFitTranformer()) @@ -48,13 +48,15 @@ pipeline = make_pipeline(MyTransformer(), MyFitTranformer()) pipeline = bob.pipelines.wrap( ["sample", "checkpoint", "dask"], pipeline, - model_path=model_path, + model_path=os.path.join(model_path, "model.pickle"), + features_dir=model_path, transform_extra_arguments=(("metadata", "metadata"),), ) # Create a dask graph from a pipeline # Run the task graph in the local computer in a single tread X_transformed = pipeline.fit_transform(X_as_sample).compute(scheduler="single-threaded") + import shutil shutil.rmtree(model_path) diff --git a/doc/python/pipeline_example_dask_sge.py b/doc/python/pipeline_example_dask_sge.py index eba1709..ae07165 100644 --- a/doc/python/pipeline_example_dask_sge.py +++ b/doc/python/pipeline_example_dask_sge.py @@ -28,7 +28,7 @@ class MyFitTranformer(TransformerMixin, BaseEstimator): def __init__(self): self._fit_model = None - def transform(self, X): + def transform(self, X, metadata=None): # Transform `X` return [x @ self._fit_model for x in X] @@ -43,7 +43,7 @@ X = numpy.zeros((2, 2)) X_as_sample = [Sample(X, key=str(i), metadata=1) for i in range(10)] # Building an arbitrary pipeline -model_path = "~/dask_tmp" +model_path = "./dask_tmp" os.makedirs(model_path, exist_ok=True) pipeline = make_pipeline(MyTransformer(), MyFitTranformer()) @@ -51,7 +51,8 @@ pipeline = make_pipeline(MyTransformer(), MyFitTranformer()) pipeline = bob.pipelines.wrap( ["sample", "checkpoint", "dask"], pipeline, - model_path=model_path, + model_path=os.path.join(model_path, "model.pickle"), + features_dir=model_path, transform_extra_arguments=(("metadata", "metadata"),), ) @@ -62,6 +63,7 @@ client = Client(cluster) # Creating the scheduler # Run the task graph in the local computer in a single tread X_transformed = pipeline.fit_transform(X_as_sample).compute(scheduler=client) + import shutil shutil.rmtree(model_path) diff --git a/doc/python/pipeline_example_dask_sge_adaptive.py b/doc/python/pipeline_example_dask_sge_adaptive.py index 5f9e791..5fab7e9 100644 --- a/doc/python/pipeline_example_dask_sge_adaptive.py +++ b/doc/python/pipeline_example_dask_sge_adaptive.py @@ -28,7 +28,7 @@ class MyFitTranformer(TransformerMixin, BaseEstimator): def __init__(self): self._fit_model = None - def transform(self, X): + def transform(self, X, metadata=None): # Transform `X` return [x @ self._fit_model for x in X] @@ -43,7 +43,7 @@ X = numpy.zeros((2, 2)) X_as_sample = [Sample(X, key=str(i), metadata=1) for i in range(10)] # Building an arbitrary pipeline -model_path = "~/dask_tmp" +model_path = "./dask_tmp" os.makedirs(model_path, exist_ok=True) pipeline = make_pipeline(MyTransformer(), MyFitTranformer()) -- GitLab