Commit bff5661a authored by Samuel GAIST's avatar Samuel GAIST Committed by Samuel GAIST

[backend][management][commands] Add GPU environment to qsetup

This will allow to properly install and run the GPU example.
parent 6203cc2d
Pipeline #30290 passed with stage
in 16 minutes and 11 seconds
......@@ -27,119 +27,124 @@
import sys
import logging
logger = logging.getLogger('beat.web')
import simplejson
from django.core.management.base import BaseCommand, CommandError
from ...models import Environment, Queue, Worker, Slot
from ...utils import setup_backend, dump_backend
# Default configuration to start the state with
import psutil
import socket
from django.core.management.base import BaseCommand
from ...utils import setup_backend, dump_backend
logger = logging.getLogger("beat.web")
CORES = psutil.cpu_count()
RAM = int(psutil.virtual_memory().total/(1024*1024))
ENVIRONMENT = {'name': 'Python 2.7', 'version': '1.3.0'}
CXX_ENVIRONMENT = {'name': 'Cxx backend', 'version': '1.1.0'}
ENVKEY = '%(name)s (%(version)s)' % ENVIRONMENT
CXX_ENVKEY = '%(name)s (%(version)s)' % CXX_ENVIRONMENT
RAM = int(psutil.virtual_memory().total / (1024 * 1024))
ENVIRONMENT = {"name": "Python 2.7", "version": "1.3.0"}
CXX_ENVIRONMENT = {"name": "Cxx backend", "version": "1.1.0"}
GPU_ENVIRONMENT = {"name": "Pytorch 0.4.0", "version": "0.0.1"}
ENVKEY = "%(name)s (%(version)s)" % ENVIRONMENT
CXX_ENVKEY = "%(name)s (%(version)s)" % CXX_ENVIRONMENT
GPU_ENVKEY = "%(name)s (%(version)s)" % GPU_ENVIRONMENT
HOSTNAME = socket.gethostname()
DEFAULT_CONFIGURATION = {
"queues": {
"queue": {
"memory-limit": int(RAM/CORES),
"time-limit": 1440, #1 day
"memory-limit": int(RAM / CORES),
"time-limit": 1440, # 1 day
"cores-per-slot": 1,
"max-slots-per-user": CORES,
"environments": [CXX_ENVKEY, ENVKEY],
"slots": {
HOSTNAME: {
"quantity": CORES,
"priority": 0
}
},
"groups": [
"Default",
],
"environments": [ENVKEY, CXX_ENVKEY, GPU_ENVKEY],
"slots": {HOSTNAME: {"quantity": CORES, "priority": 0}},
"groups": ["Default"],
}
},
"environments": {
ENVKEY: {
"name": ENVIRONMENT['name'],
"version": ENVIRONMENT['version'],
"languages": ['python'],
"name": ENVIRONMENT["name"],
"version": ENVIRONMENT["version"],
"languages": ["python"],
"short_description": "Local python interpreter",
"description": "Automatically generated local python " \
"description": "Automatically generated local python "
"interpreter environment",
},
CXX_ENVKEY: {
"name": CXX_ENVIRONMENT['name'],
"version": CXX_ENVIRONMENT['version'],
"languages": ['cxx'],
CXX_ENVKEY: {
"name": CXX_ENVIRONMENT["name"],
"version": CXX_ENVIRONMENT["version"],
"languages": ["cxx"],
"short_description": "C++ backend",
"description": "C++ backend running in a docker container",
},
GPU_ENVKEY: {
"name": GPU_ENVIRONMENT["name"],
"version": GPU_ENVIRONMENT["version"],
"languages": ["python"],
"short_description": "Python GPU backend",
"description": "Python backend running in a docker container with GPU support",
},
},
"workers": {
HOSTNAME: {
"cores": CORES,
"memory": RAM
}
}
"workers": {HOSTNAME: {"cores": CORES, "memory": RAM}},
}
RESET_CONFIGURATION = {
"queues": {},
"environments": {},
"workers": {}
}
RESET_CONFIGURATION = {"queues": {}, "environments": {}, "workers": {}}
class Command(BaseCommand):
help = 'Sets and resets queue configurations'
help = "Sets and resets queue configurations"
def add_arguments(self, parser):
parser.add_argument('--dump', action='store_true', dest='dump',
default=False, help='Dump all environment/worker/queues ' \
'configuration and exits')
parser.add_argument('--reset', action='store_true', dest='reset',
default=False, help='Delete all environment/worker/queues ' \
'before setting the given configuration')
parser.add_argument('config', type=str, nargs='?',
help='Optional custom queue configuration to use. If not ' \
'passed, uses an internal default with a single ' \
'queue/worker/%d slots' % CORES)
parser.add_argument(
"--dump",
action="store_true",
dest="dump",
default=False,
help="Dump all environment/worker/queues " "configuration and exits",
)
parser.add_argument(
"--reset",
action="store_true",
dest="reset",
default=False,
help="Delete all environment/worker/queues "
"before setting the given configuration",
)
parser.add_argument(
"config",
type=str,
nargs="?",
help="Optional custom queue configuration to use. If not "
"passed, uses an internal default with a single "
"queue/worker/%d slots" % CORES,
)
def handle(self, *ignored, **arguments):
# Setup this command's logging level
global logger
arguments['verbosity'] = int(arguments['verbosity'])
if arguments['verbosity'] >= 1:
if arguments['verbosity'] == 1: logger.setLevel(logging.INFO)
elif arguments['verbosity'] >= 2: logger.setLevel(logging.DEBUG)
if arguments['dump']:
d=dump_backend()
arguments["verbosity"] = int(arguments["verbosity"])
if arguments["verbosity"] >= 1:
if arguments["verbosity"] == 1:
logger.setLevel(logging.INFO)
elif arguments["verbosity"] >= 2:
logger.setLevel(logging.DEBUG)
if arguments["dump"]:
d = dump_backend()
print(simplejson.dumps(d, indent=2))
sys.exit(0)
if arguments['reset']:
if arguments["reset"]:
setup_backend(RESET_CONFIGURATION)
config = None
if arguments['config']:
with open(arguments['config'], 'rb') as f:
if arguments["config"]:
with open(arguments["config"], "rb") as f:
config = simplejson.load(f)
setup_backend(config or DEFAULT_CONFIGURATION)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment