Skip to content
Snippets Groups Projects
Commit ad9d432b authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira
Browse files

Project variable

parent 7c63ca0c
No related branches found
No related tags found
No related merge requests found
from bob.pipelines.distributed.sge import SGEMultipleQueuesCluster
from dask.distributed import Client
cluster = SGEMultipleQueuesCluster()
cluster = SGEMultipleQueuesCluster(min_jobs=20)
dask_client = Client(cluster)
from bob.pipelines.distributed.sge import SGEMultipleQueuesCluster
from bob.pipelines.distributed.sge_queues import QUEUE_DEMANDING
from dask.distributed import Client
cluster = SGEMultipleQueuesCluster(min_jobs=20, sge_job_spec=QUEUE_DEMANDING)
dask_client = Client(cluster)
......@@ -2,5 +2,5 @@ from bob.pipelines.distributed.sge import SGEMultipleQueuesCluster
from bob.pipelines.distributed.sge_queues import QUEUE_LIGHT
from dask.distributed import Client
cluster = SGEMultipleQueuesCluster(sge_job_spec=QUEUE_LIGHT)
cluster = SGEMultipleQueuesCluster(min_jobs=20, sge_job_spec=QUEUE_LIGHT)
dask_client = Client(cluster)
......@@ -18,7 +18,7 @@ from distributed.scheduler import Scheduler
from distributed.deploy import Adaptive
from .sge_queues import QUEUE_DEFAULT
from bob.extension import rc
class SGEIdiapJob(Job):
"""Launches a SGE Job in the IDIAP cluster. This class basically encodes
......@@ -37,12 +37,13 @@ class SGEIdiapJob(Job):
self,
*args,
queue=None,
project="biometric",
project=rc.get("sge.project"),
resource_spec=None,
job_extra=None,
config_name="sge",
**kwargs,
):
if queue is None:
queue = dask.config.get("jobqueue.%s.queue" % config_name)
if project is None:
......@@ -206,7 +207,9 @@ class SGEMultipleQueuesCluster(JobQueueCluster):
env_extra=None,
sge_job_spec=QUEUE_DEFAULT,
min_jobs=10,
project=rc.get('sge.project'),
**kwargs,
):
# Defining the job launcher
......@@ -215,6 +218,7 @@ class SGEMultipleQueuesCluster(JobQueueCluster):
self.protocol = protocol
self.log_directory = log_directory
self.project = project
silence_logs = "error"
secutity = None
......@@ -222,6 +226,7 @@ class SGEMultipleQueuesCluster(JobQueueCluster):
host = None
security = None
if env_extra is None:
env_extra = []
elif not isinstance(env_extra, list):
......@@ -290,6 +295,7 @@ class SGEMultipleQueuesCluster(JobQueueCluster):
return {
"queue": queue,
"project": self.project,
"memory": _get_key_from_spec(job_spec, "memory"),
"cores": 1,
"processes": 1,
......
......@@ -72,6 +72,14 @@ Below follow a nice video explaining what is the :doc:`Dask-Jobqueue <dask-jobqu
<iframe width="560" height="315" src="https://www.youtube.com/embed/FXsgmwpRExM" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
.. warning::
To submit jobs at Idiap's SGE it's mandatory to set a project name. Run the code below to set it::
$ bob config set sge.project <YOUR_PROJECT_NAME>
The snippet below shows how to deploy the exact same pipeline from the previous section in the Idiap SGE cluster
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment