Merge branch 'sge-defaults' into 'master'

Project variable

See merge request !34

bob/pipelines/config/distributed/sge_default.py

 from bob.pipelines.distributed.sge import SGEMultipleQueuesCluster
 from dask.distributed import Client
 
-cluster = SGEMultipleQueuesCluster()
+cluster = SGEMultipleQueuesCluster(min_jobs=20)
 dask_client = Client(cluster)

bob/pipelines/config/distributed/sge_demanding.py

+from bob.pipelines.distributed.sge import SGEMultipleQueuesCluster
+from bob.pipelines.distributed.sge_queues import QUEUE_DEMANDING
+from dask.distributed import Client
+
+cluster = SGEMultipleQueuesCluster(min_jobs=20, sge_job_spec=QUEUE_DEMANDING)
+dask_client = Client(cluster)

bob/pipelines/config/distributed/sge_light.py

@@ -2,5 +2,5 @@ from bob.pipelines.distributed.sge import SGEMultipleQueuesCluster
 from bob.pipelines.distributed.sge_queues import QUEUE_LIGHT
 from dask.distributed import Client
 
-cluster = SGEMultipleQueuesCluster(sge_job_spec=QUEUE_LIGHT)
+cluster = SGEMultipleQueuesCluster(min_jobs=20, sge_job_spec=QUEUE_LIGHT)
 dask_client = Client(cluster)

bob/pipelines/distributed/sge.py

@@ -18,7 +18,7 @@ from distributed.scheduler import Scheduler
 from distributed.deploy import Adaptive
 
 from .sge_queues import QUEUE_DEFAULT
-
+from bob.extension import rc
 
 class SGEIdiapJob(Job):
     """Launches a SGE Job in the IDIAP cluster. This class basically encodes
@@ -37,12 +37,13 @@ class SGEIdiapJob(Job):
         self,
         *args,
         queue=None,
-        project="biometric",
+        project=rc.get("sge.project"),
         resource_spec=None,
         job_extra=None,
         config_name="sge",
         **kwargs,
     ):
+
         if queue is None:
             queue = dask.config.get("jobqueue.%s.queue" % config_name)
         if project is None:
@@ -206,7 +207,9 @@ class SGEMultipleQueuesCluster(JobQueueCluster):
         env_extra=None,
         sge_job_spec=QUEUE_DEFAULT,
         min_jobs=10,
+        project=rc.get('sge.project'),
         **kwargs,
+
     ):
 
         # Defining the job launcher
@@ -215,6 +218,7 @@ class SGEMultipleQueuesCluster(JobQueueCluster):
 
         self.protocol = protocol
         self.log_directory = log_directory
+        self.project = project
 
         silence_logs = "error"
         secutity = None
@@ -222,6 +226,7 @@ class SGEMultipleQueuesCluster(JobQueueCluster):
         host = None
         security = None
 
+
         if env_extra is None:
             env_extra = []
         elif not isinstance(env_extra, list):
@@ -290,6 +295,7 @@ class SGEMultipleQueuesCluster(JobQueueCluster):
 
         return {
             "queue": queue,
+            "project": self.project,
             "memory": _get_key_from_spec(job_spec, "memory"),
             "cores": 1,
             "processes": 1,

doc/dask.rst

@@ -72,6 +72,14 @@ Below follow a nice video explaining what is the :doc:`Dask-Jobqueue <dask-jobqu
      <iframe width="560" height="315" src="https://www.youtube.com/embed/FXsgmwpRExM" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen></iframe>
 
 
+.. warning:: 
+
+  To submit jobs at Idiap's SGE it's mandatory to set a project name. Run the code below to set it::
+
+  $ bob config set sge.project <YOUR_PROJECT_NAME>
+
+
+
 The snippet below shows how to deploy the exact same pipeline from the previous section in the Idiap SGE cluster