Commit 18961cfd authored by Samuel GAIST's avatar Samuel GAIST Committed by Flavio TARSETTI
Browse files

[backend][all] Pre-commit cleanup

parent 79349c07
Pipeline #42562 passed with stage
in 16 minutes and 18 seconds
......@@ -25,43 +25,37 @@
# #
###############################################################################
from django.contrib import admin
from django import forms
from django.contrib import admin
from ..common.admin import Django18ProofGuardedModelAdmin
from ..common.admin import notify_by_email
from ..common.texts import Messages
from ..ui.forms import CodeMirrorRSTCharField
from .models import Environment as EnvironmentModel
from .models import EnvironmentLanguage as EnvironmentLanguageModel
from .models import Worker as WorkerModel
from .models import Queue as QueueModel
from .models import Slot as SlotModel
from .models import Job as JobModel
from .models import JobSplit as JobSplitModel
from .models import Queue as QueueModel
from .models import Slot as SlotModel
from .models import Worker as WorkerModel
from ..ui.forms import CodeMirrorRSTCharField
from ..common.texts import Messages
from ..common.admin import Django18ProofGuardedModelAdmin, notify_by_email
# ----------------------------------------------------------
#----------------------------------------------------------
class EnvironmentModelForm(forms.ModelForm):
description = CodeMirrorRSTCharField(
required=False,
help_text=Messages['description'],
required=False, help_text=Messages["description"],
)
class Meta:
model = EnvironmentModel
exclude = []
widgets = {
'name': forms.TextInput(
attrs=dict(size=100),
),
'version': forms.TextInput(
attrs=dict(size=40),
),
'short_description': forms.TextInput(
attrs=dict(size=100),
),
"name": forms.TextInput(attrs=dict(size=100),),
"version": forms.TextInput(attrs=dict(size=40),),
"short_description": forms.TextInput(attrs=dict(size=100),),
}
......@@ -71,66 +65,57 @@ class EnvironmentLanguageInline(admin.TabularInline):
class Environment(admin.ModelAdmin):
list_display = (
'id',
'name',
'version',
'sharing',
'active',
'short_description',
list_display = (
"id",
"name",
"version",
"sharing",
"active",
"short_description",
)
search_fields = [
'name',
'version',
'short_description',
'description',
search_fields = [
"name",
"version",
"short_description",
"description",
]
list_display_links = (
'id',
'name',
list_display_links = (
"id",
"name",
)
inlines = [
EnvironmentLanguageInline
]
inlines = [EnvironmentLanguageInline]
form = EnvironmentModelForm
filter_horizontal = [
'shared_with',
'shared_with_team'
]
filter_horizontal = ["shared_with", "shared_with_team"]
actions = [
notify_by_email('environment_notifications_enabled'),
notify_by_email("environment_notifications_enabled"),
]
fieldsets = (
(None,
dict(
fields=('name', 'version', 'previous_version', 'active'),
),
),
('Documentation',
dict(
classes=('collapse',),
fields=('short_description', 'description',),
),
),
('Sharing',
dict(
classes=('collapse',),
fields=('sharing', 'shared_with', 'shared_with_team'),
),
),
(None, dict(fields=("name", "version", "previous_version", "active"),),),
(
"Documentation",
dict(classes=("collapse",), fields=("short_description", "description",),),
),
(
"Sharing",
dict(
classes=("collapse",),
fields=("sharing", "shared_with", "shared_with_team"),
),
),
)
admin.site.register(EnvironmentModel, Environment)
#----------------------------------------------------------
# ----------------------------------------------------------
def activate_workers(modeladmin, request, queryset):
......@@ -139,7 +124,9 @@ def activate_workers(modeladmin, request, queryset):
for q in queryset:
q.active = True
q.save()
activate_workers.short_description = 'Activate selected workers'
activate_workers.short_description = "Activate selected workers"
def deactivate_workers(modeladmin, request, queryset):
......@@ -148,44 +135,55 @@ def deactivate_workers(modeladmin, request, queryset):
for q in queryset:
q.active = False
q.save()
deactivate_workers.short_description = 'Deactivate selected workers'
deactivate_workers.short_description = "Deactivate selected workers"
class Worker(admin.ModelAdmin):
list_display = ('id', 'name', 'cores', 'memory', 'active')
search_fields = ['name']
list_display_links = ('id', 'name')
list_display = ("id", "name", "cores", "memory", "active")
search_fields = ["name"]
list_display_links = ("id", "name")
actions = [
activate_workers,
deactivate_workers,
]
admin.site.register(WorkerModel, Worker)
#----------------------------------------------------------
# ----------------------------------------------------------
class SlotInline(admin.TabularInline):
model = SlotModel
#----------------------------------------------------------
# ----------------------------------------------------------
class Queue(Django18ProofGuardedModelAdmin):
list_display = ('id', 'name', 'memory_limit', 'time_limit', 'cores_per_slot', 'max_slots_per_user')
search_fields = ['name']
list_display_links = ('id', 'name')
inlines = [SlotInline]
list_display = (
"id",
"name",
"memory_limit",
"time_limit",
"cores_per_slot",
"max_slots_per_user",
)
search_fields = ["name"]
list_display_links = ("id", "name")
inlines = [SlotInline]
admin.site.register(QueueModel, Queue)
#----------------------------------------------------------
# ----------------------------------------------------------
class JobSplitInline(admin.TabularInline):
......@@ -195,25 +193,25 @@ class JobSplitInline(admin.TabularInline):
return False
def has_add_permission(self, request):
return False
return False
class Job(admin.ModelAdmin):
list_display = ('id', 'key', 'runnable_date', 'start_date', 'block', 'splits')
search_fields = ['block__name', 'block__experiment__name']
list_display_links = ('id', 'block', 'key')
ordering = ('runnable_date', 'start_date', 'id')
inlines = [JobSplitInline]
list_display = ("id", "key", "runnable_date", "start_date", "block", "splits")
search_fields = ["block__name", "block__experiment__name"]
list_display_links = ("id", "block", "key")
ordering = ("runnable_date", "start_date", "id")
inlines = [JobSplitInline]
# to avoid very slow loading of cached files
raw_id_fields = ('block',)
raw_id_fields = ("block",)
def splits(self, obj):
return obj.splits.count()
def has_add_permission(self, request):
return False
return False
admin.site.register(JobModel, Job)
......@@ -25,109 +25,114 @@
# #
###############################################################################
from rest_framework.decorators import api_view, permission_classes
from rest_framework.response import Response
from django.conf import settings
from rest_framework import permissions
from rest_framework import status
from django.conf import settings
from .models import Environment
from .models import Worker
from .models import LocalSchedulerProcesses
from rest_framework.decorators import api_view
from rest_framework.decorators import permission_classes
from rest_framework.response import Response
from ..code.models import Code
from . import local_scheduler
from .models import Environment
from .models import LocalSchedulerProcesses
from .models import Worker
#----------------------------------------------------------
# ----------------------------------------------------------
@api_view(['GET'])
@api_view(["GET"])
@permission_classes([permissions.AllowAny])
def accessible_environments_list(request):
"""Returns all accessible environments for a given user"""
# Retrieve the list of environments
environments = Environment.objects.filter(active=True).order_by('name',
'version')
environments = Environment.objects.filter(active=True).order_by("name", "version")
result = []
for environment in environments.iterator():
# Check that the user has access to the environment
(has_access, accessibility) = environment.accessibility_for(request.user)
if not(has_access):
if not (has_access):
continue
# Retrieve the details about the queues
queues = {}
for queue in environment.queues.iterator():
if request.user.has_perm('can_access', queue):
if request.user.has_perm("can_access", queue):
queues[queue.name] = {
'nb_slots': queue.number_of_slots(),
'memory_limit': queue.memory_limit,
'time_limit': queue.time_limit,
'nb_cores_per_slot': queue.cores_per_slot,
'max_slots_per_user':queue.max_slots_per_user,
"nb_slots": queue.number_of_slots(),
"memory_limit": queue.memory_limit,
"time_limit": queue.time_limit,
"nb_cores_per_slot": queue.cores_per_slot,
"max_slots_per_user": queue.max_slots_per_user,
}
# Only returns environments which effectively have queues associated
if environment.queues.count():
result.append({
'name': environment.name,
'version': environment.version,
'short_description': environment.short_description,
'queues': queues,
'accessibility': accessibility,
'languages': [ Code.language_identifier(x.language) for x in environment.languages.iterator() ],
})
result.append(
{
"name": environment.name,
"version": environment.version,
"short_description": environment.short_description,
"queues": queues,
"accessibility": accessibility,
"languages": [
Code.language_identifier(x.language)
for x in environment.languages.iterator()
],
}
)
return Response(result)
#----------------------------------------------------------
# ----------------------------------------------------------
@api_view(['POST'])
@api_view(["POST"])
@permission_classes([permissions.IsAdminUser])
def start_local_scheduler(request):
"""Starts the local scheduler"""
if not getattr(settings, 'SCHEDULING_PANEL', False):
if not getattr(settings, "SCHEDULING_PANEL", False):
return Response(status=status.HTTP_403_FORBIDDEN)
# Clean start-up
LocalSchedulerProcesses.objects.all().delete()
address = getattr(settings, 'LOCAL_SCHEDULER_ADDRESS', '127.0.0.1')
port = getattr(settings, 'LOCAL_SCHEDULER_PORT', 50000)
use_docker = getattr(settings, 'LOCAL_SCHEDULER_USE_DOCKER', False)
address = getattr(settings, "LOCAL_SCHEDULER_ADDRESS", "127.0.0.1")
port = getattr(settings, "LOCAL_SCHEDULER_PORT", 50000)
use_docker = getattr(settings, "LOCAL_SCHEDULER_USE_DOCKER", False)
full_address = 'tcp://%s:%d' % (address, port)
full_address = "tcp://%s:%d" % (address, port)
for worker in Worker.objects.all():
(pid, _) = local_scheduler.start_worker(worker.name, settings.PREFIX,
settings.CACHE_ROOT,
full_address,
use_docker=use_docker)
(pid, _) = local_scheduler.start_worker(
worker.name,
settings.PREFIX,
settings.CACHE_ROOT,
full_address,
use_docker=use_docker,
)
LocalSchedulerProcesses(name=worker.name, pid=pid).save()
(pid, _) = local_scheduler.start_scheduler(address=address, port=port)
LocalSchedulerProcesses(name='Scheduler', pid=pid).save()
LocalSchedulerProcesses(name="Scheduler", pid=pid).save()
return Response(status=status.HTTP_204_NO_CONTENT)
#----------------------------------------------------------
# ----------------------------------------------------------
@api_view(['POST'])
@api_view(["POST"])
@permission_classes([permissions.IsAdminUser])
def stop_local_scheduler(request):
"""Starts the local scheduler"""
if not getattr(settings, 'SCHEDULING_PANEL', False):
if not getattr(settings, "SCHEDULING_PANEL", False):
return Response(status=status.HTTP_403_FORBIDDEN)
for process in LocalSchedulerProcesses.objects.all():
......
......@@ -28,7 +28,8 @@
from django.apps import AppConfig
from django.utils.translation import ugettext_lazy as _
class BackendConfig(AppConfig):
name = 'beat.web.backend'
verbose_name = _('Backend')
name = "beat.web.backend"
verbose_name = _("Backend")
This diff is collapsed.
......@@ -26,73 +26,72 @@
###############################################################################
from django.conf import settings
from django import db
import multiprocessing
import psutil
import signal
from ..scripts import scheduler
import psutil
from django import db
from django.conf import settings
from beat.core.scripts import worker
from ..scripts import scheduler
#----------------------------------------------------------
# ----------------------------------------------------------
class SchedulerProcess(multiprocessing.Process):
def __init__(self, arguments, queue=None):
super(SchedulerProcess, self).__init__()
def __init__(self, arguments, queue=None):
super(SchedulerProcess, self).__init__()
if queue is None:
self.queue = multiprocessing.Queue()
else:
self.queue = queue
if queue is None:
self.queue = multiprocessing.Queue()
else:
self.queue = queue
self.arguments = arguments
self.arguments = arguments
def run(self):
self.queue.put("STARTED")
scheduler.main(self.arguments)
def run(self):
self.queue.put('STARTED')
scheduler.main(self.arguments)
#----------------------------------------------------------
# ----------------------------------------------------------
class WorkerProcess(multiprocessing.Process):
def __init__(self, arguments, queue=None):
super(WorkerProcess, self).__init__()
def __init__(self, arguments, queue=None):
super(WorkerProcess, self).__init__()
if queue is None:
self.queue = multiprocessing.Queue()
else:
self.queue = queue
self.arguments = arguments
if queue is None:
self.queue = multiprocessing.Queue()
else:
self.queue = queue
self.arguments = arguments
def run(self):
self.queue.put('STARTED')
worker.main(self.arguments)
def run(self):
self.queue.put("STARTED")
worker.main(self.arguments)
#----------------------------------------------------------
# ----------------------------------------------------------
def start_scheduler(settings_module='beat.web.settings.settings', interval=5,
address='127.0.0.1', port=50000):
def start_scheduler(
settings_module="beat.web.settings.settings",
interval=5,
address="127.0.0.1",
port=50000,
):
args = [
'--settings=%s' % str(settings_module),
'--interval=%d' % int(interval),
'--address=%s' % str(address),
'--port=%d' % int(port),
"--settings=%s" % str(settings_module),
"--interval=%d" % int(interval),
"--address=%s" % str(address),
"--port=%d" % int(port),
]
if getattr(settings, 'LOCAL_SCHEDULER_VERBOSITY', None) is not None:
if getattr(settings, "LOCAL_SCHEDULER_VERBOSITY", None) is not None:
args.append(settings.LOCAL_SCHEDULER_VERBOSITY)
db.connections.close_all()
......@@ -104,21 +103,21 @@ def start_scheduler(settings_module='beat.web.settings.settings', interval=5,
return (process.pid, process)
#----------------------------------------------------------
# ----------------------------------------------------------
def start_worker(name, prefix, cache, scheduler_address, use_docker=False):
args = [
'--prefix=%s' % str(prefix),
'--cache=%s' % str(cache),
'--name=%s' % str(name),
str(scheduler_address)
"--prefix=%s" % str(prefix),
"--cache=%s" % str(cache),
"--name=%s" % str(name),
str(scheduler_address),
]
if use_docker:
args.insert(3, '--docker')
args.insert(3, "--docker")
if getattr(settings, 'LOCAL_SCHEDULER_VERBOSITY', None) is not None:
if getattr(settings, "LOCAL_SCHEDULER_VERBOSITY", None) is not None:
args.insert(3, settings.LOCAL_SCHEDULER_VERBOSITY)
process = WorkerProcess(args)
......@@ -128,7 +127,7 @@ def start_worker(name, prefix, cache, scheduler_address, use_docker=False):
return (process.pid, process)
#----------------------------------------------------------
# ----------------------------------------------------------
def stop_process(pid):
......
......@@ -25,24 +25,21 @@
# #
###############################################################################
'''Utilities for summarizing scheduler state'''
import os
"""Utilities for summarizing scheduler state"""
import logging
logger = logging.getLogger(__name__)
import os
import psutil
from django.conf import settings
from .models import Job, JobSplit
from ..experiments.models import Experiment
from .models import JobSplit
logger = logging.getLogger(__name__)
def cache():