Commit 1a11a53a authored by Tiago de Freitas Pereira's avatar Tiago de Freitas Pereira

Merge branch 'adaptive-qsub' into 'master'

Adaptive and heterogenous resource allocation

See merge request !21
parents 8bcdefc3 a0556bdb
Pipeline #38820 failed with stages
in 3 minutes and 32 seconds
......@@ -17,4 +17,5 @@ build
.coverage
record.txt
miniconda.sh
miniconda/
\ No newline at end of file
miniconda/
*.DS_Store
from bob.pipelines.distributed.sge import SGEIdiapCluster
from dask.distributed import Client
Q_1DAY_IO_BIG_SPEC = {
"default": {
"queue": "q_1day",
"memory": "4GB",
"io_big": True,
"resource_spec": "",
"resources": "",
}
}
n_jobs = 16
cluster = SGEIdiapCluster()
cluster.scale(n_jobs, queue="q_1day", io_big=True)
cluster = SGEIdiapCluster(sge_job_spec=Q_1DAY_IO_BIG_SPEC)
cluster.scale(n_jobs)
dask_client = Client(cluster)
from bob.pipelines.distributed.sge import SGEIdiapCluster
from bob.pipelines.distributed.sge import SGEIdiapCluster, Q_1DAY_GPU_SPEC
from dask.distributed import Client
n_jobs = 16
n_gpu_jobs = 1
cluster = SGEIdiapCluster()
cluster.scale(n_jobs, queue="q_1day", io_big=True)
cluster.scale(n_gpu_jobs, queue="q_gpu", resources="GPU=1")
cluster = SGEIdiapCluster(sge_job_spec=Q_1DAY_GPU_SPEC)
cluster.scale(n_jobs, sge_job_spec_key="default")
cluster.scale(n_gpu_jobs, sge_job_spec_key="gpu")
dask_client = Client(cluster)
from bob.pipelines.distributed.sge import SGEIdiapCluster
from bob.pipelines.distributed.sge import SGEIdiapCluster, Q_1DAY_IO_BIG_SPEC
from dask.distributed import Client
n_jobs = 48
cluster = SGEIdiapCluster()
cluster.scale(n_jobs, queue="q_1day", io_big=True, memory="8GB")
cluster = SGEIdiapCluster(sge_job_spec=Q_1DAY_IO_BIG_SPEC)
cluster.scale(n_jobs)
dask_client = Client(cluster)
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment