Skip to content
Snippets Groups Projects
Commit 0e7d7bb4 authored by Samuel GAIST's avatar Samuel GAIST Committed by Flavio TARSETTI
Browse files

[experiments][all] Pre-commit cleanup

parent d9b54f85
No related branches found
No related tags found
2 merge requests!353Cleanup experiments,!342Django 3 migration
Pipeline #42589 passed
Stage: build
Hide whitespace changes
Inline Side-by-side
beat/web/experiments/admin.py
+ 272
− 288
View file @ 0e7d7bb4
...@@ -26,77 +26,72 @@ ...@@ -26,77 +26,72 @@
############################################################################### ###############################################################################
import simplejson as json import simplejson as json
from django import forms from django import forms
from django.contrib import admin from django.contrib import admin
from django.core.files.base import ContentFile from django.core.files.base import ContentFile
from django.db.models import Count
from django.db.models import Max
from django.urls import reverse
from django.utils.html import format_html from django.utils.html import format_html
from django.utils.safestring import mark_safe from django.utils.safestring import mark_safe
from django.urls import reverse
from django.db.models import Max, Count
from .models import Experiment as ExperimentModel from ..common.texts import Messages
from ..ui.forms import CodeMirrorJSONCharField
from ..ui.forms import CodeMirrorJSONFileField
from ..ui.forms import CodeMirrorRSTFileField
from ..ui.forms import NameField
from .models import Block as BlockModel from .models import Block as BlockModel
from .models import Result as ResultModel
from .models import CachedFile as CachedFileModel
from .models import BlockInput as BlockInputModel from .models import BlockInput as BlockInputModel
from .models import CachedFile as CachedFileModel
from .models import Experiment as ExperimentModel
from .models import Result as ResultModel
from .models import validate_experiment from .models import validate_experiment
from ..ui.forms import CodeMirrorJSONFileField, CodeMirrorRSTFileField, \ # ----------------------------------------------------------
NameField, CodeMirrorJSONCharField
from ..common.texts import Messages
#----------------------------------------------------------
class ExperimentModelForm(forms.ModelForm): class ExperimentModelForm(forms.ModelForm):
name = NameField( name = NameField(
widget=forms.TextInput(attrs=dict(size=80)), widget=forms.TextInput(attrs=dict(size=80)), help_text=Messages["name"],
help_text=Messages['name'],
) )
declaration_file = CodeMirrorJSONFileField( declaration_file = CodeMirrorJSONFileField(
label='Declaration', label="Declaration", help_text=Messages["json"],
help_text=Messages['json'],
) )
description_file = CodeMirrorRSTFileField( description_file = CodeMirrorRSTFileField(
label='Description', label="Description",
required=False, required=False,
allow_empty_file=True, allow_empty_file=True,
help_text=Messages['description'], help_text=Messages["description"],
) )
class Meta: class Meta:
model = ExperimentModel model = ExperimentModel
exclude = [] exclude = []
widgets = { widgets = {
'short_description': forms.TextInput( "short_description": forms.TextInput(attrs=dict(size=100),),
attrs=dict(size=100),
),
} }
def clean_declaration_file(self): def clean_declaration_file(self):
"""Cleans-up the declaration_file data, make sure it is really new""" """Cleans-up the declaration_file data, make sure it is really new"""
new_declaration = self.cleaned_data['declaration_file'].read() new_declaration = self.cleaned_data["declaration_file"].read()
old_declaration = '' old_declaration = ""
if self.instance and self.instance.declaration_file.name is not None: if self.instance and self.instance.declaration_file.name is not None:
old_declaration = self.instance.declaration_string old_declaration = self.instance.declaration_string
if new_declaration == old_declaration: if new_declaration == old_declaration:
self.changed_data.remove('declaration_file') self.changed_data.remove("declaration_file")
content_file = ContentFile(old_declaration) content_file = ContentFile(old_declaration)
content_file.name = self.instance.declaration_file.name content_file.name = self.instance.declaration_file.name
return content_file return content_file
try: try:
core_experiment, errors = \ core_experiment, errors = validate_experiment(
validate_experiment(json.loads(new_declaration), json.loads(new_declaration), self.cleaned_data["toolchain"].declaration
self.cleaned_data['toolchain'].declaration) )
except SyntaxError as e: except SyntaxError as e:
raise forms.ValidationError(str(e)) raise forms.ValidationError(str(e))
...@@ -104,84 +99,98 @@ class ExperimentModelForm(forms.ModelForm): ...@@ -104,84 +99,98 @@ class ExperimentModelForm(forms.ModelForm):
all_errors = [forms.ValidationError(k) for k in errors] all_errors = [forms.ValidationError(k) for k in errors]
raise forms.ValidationError(all_errors) raise forms.ValidationError(all_errors)
self.cleaned_data['declaration_file'].seek(0) #reset ContentFile readout self.cleaned_data["declaration_file"].seek(0) # reset ContentFile readout
return self.cleaned_data['declaration_file'] return self.cleaned_data["declaration_file"]
def clean(self): def clean(self):
"""Cleans-up the input data, make sure it overall validates""" """Cleans-up the input data, make sure it overall validates"""
if 'declaration_file' in self.data and \ if "declaration_file" in self.data and isinstance(
isinstance(self.data['declaration_file'], str): self.data["declaration_file"], str
):
mutable_data = self.data.copy() mutable_data = self.data.copy()
mutable_data['declaration_file'] = ContentFile(self.data['declaration_file'], name='unsaved') mutable_data["declaration_file"] = ContentFile(
self.data["declaration_file"], name="unsaved"
)
self.data = mutable_data self.data = mutable_data
class BlockInline(admin.TabularInline): class BlockInline(admin.TabularInline):
model = BlockModel model = BlockModel
extra = 0 extra = 0
readonly_fields = ['execution_order', 'link', 'algorithm', 'analyzer', readonly_fields = ["execution_order", "link", "algorithm", "analyzer", "status"]
'status'] ordering = ["execution_order"]
ordering = ['execution_order']
fields = readonly_fields fields = readonly_fields
def link(self, obj): def link(self, obj):
url = reverse('admin:experiments_block_change', args=(obj.pk,)) url = reverse("admin:experiments_block_change", args=(obj.pk,))
return mark_safe('<a href="%s">%s</a>' % (url, obj.name)) return mark_safe('<a href="%s">%s</a>' % (url, obj.name)) # nosec
link.short_description = 'name'
link.short_description = "name"
def has_delete_permission(self, request, obj=None): def has_delete_permission(self, request, obj=None):
return False return False
def has_add_permission(self, request): def has_add_permission(self, request):
return False return False
#---------------------------------------------------------- # ----------------------------------------------------------
def reset_experiment(modeladmin, request, queryset): def reset_experiment(modeladmin, request, queryset):
for q in queryset: q.reset() for q in queryset:
reset_experiment.short_description = 'Reset selected experiments' q.reset()
reset_experiment.short_description = "Reset selected experiments"
def cancel_experiment(modeladmin, request, queryset): def cancel_experiment(modeladmin, request, queryset):
for q in queryset: q.cancel() for q in queryset:
cancel_experiment.short_description = 'Cancel selected experiments' q.cancel()
cancel_experiment.short_description = "Cancel selected experiments"
def rehash_experiment(modeladmin, request, queryset): def rehash_experiment(modeladmin, request, queryset):
for q in queryset: q.save() for q in queryset:
rehash_experiment.short_description = 'Rehash selected experiments' q.save()
rehash_experiment.short_description = "Rehash selected experiments"
class Experiment(admin.ModelAdmin): class Experiment(admin.ModelAdmin):
list_display = ('id', list_display = (
'author', "id",
'toolchain', "author",
'name', "toolchain",
'creation_date', "name",
'start_date', "creation_date",
'end_date', "start_date",
'status', "end_date",
'sharing', "status",
) "sharing",
search_fields = ['author__username', )
'toolchain__name', search_fields = [
'toolchain__author__username', "author__username",
'name', "toolchain__name",
'short_description', "toolchain__author__username",
] "name",
readonly_fields = ( "short_description",
'hash', ]
'referenced_datasets', readonly_fields = (
'referenced_algorithms', "hash",
'short_description', "referenced_datasets",
"referenced_algorithms",
"short_description",
) )
list_display_links = ('id', ) list_display_links = ("id",)
actions = [ actions = [
rehash_experiment, rehash_experiment,
...@@ -191,135 +200,126 @@ class Experiment(admin.ModelAdmin): ...@@ -191,135 +200,126 @@ class Experiment(admin.ModelAdmin):
form = ExperimentModelForm form = ExperimentModelForm
filter_horizontal = [ filter_horizontal = ["shared_with", "shared_with_team"]
'shared_with',
'shared_with_team'
]
inlines = [ inlines = [
BlockInline, BlockInline,
] ]
fieldsets = ( fieldsets = (
(None, (None, dict(fields=("name", "author", "toolchain"),),),
dict( (
fields=('name', 'author', 'toolchain'), "Status and dates",
), dict(classes=("collapse",), fields=("start_date", "end_date", "status"),),
), ),
('Status and dates', (
dict( "Documentation",
classes=('collapse',), dict(
fields=('start_date', 'end_date', 'status'), classes=("collapse",),
), fields=("short_description", "description_file",),
), ),
('Documentation', ),
dict( (
classes=('collapse',), "References (read-only)",
fields=('short_description', 'description_file',), dict(
), classes=("collapse",),
), fields=("referenced_datasets", "referenced_algorithms",),
('References (read-only)', ),
dict( ),
classes=('collapse',), (
fields=('referenced_datasets', 'referenced_algorithms',), "Sharing",
), dict(
), classes=("collapse",),
('Sharing', fields=("sharing", "shared_with", "shared_with_team"),
dict( ),
classes=('collapse',), ),
fields=('sharing', 'shared_with', 'shared_with_team'), ("Source code", dict(fields=("hash", "declaration_file"),),),
),
),
('Source code',
dict(
fields=('hash', 'declaration_file'),
),
),
) )
admin.site.register(ExperimentModel, Experiment) admin.site.register(ExperimentModel, Experiment)
#---------------------------------------------------------- # ----------------------------------------------------------
class BlockInputInline(admin.TabularInline): class BlockInputInline(admin.TabularInline):
model = BlockInputModel model = BlockInputModel
verbose_name = 'Input' verbose_name = "Input"
verbose_name_plural = 'Inputs' verbose_name_plural = "Inputs"
extra = 0 extra = 0
ordering = ['database', 'cache'] ordering = ["database", "cache"]
readonly_fields = ['input', 'channel'] readonly_fields = ["input", "channel"]
fields = readonly_fields fields = readonly_fields
def input(self, obj): def input(self, obj):
if obj.database: if obj.database:
url = reverse('admin:databases_databaseset_change', url = reverse(
args=(obj.database.set.pk,)) "admin:databases_databaseset_change", args=(obj.database.set.pk,)
text = '%s (%s)' % (obj.database, obj.database.hash) )
what = 'Dataset Output' text = "%s (%s)" % (obj.database, obj.database.hash)
what = "Dataset Output"
else: else:
url = reverse('admin:experiments_cachedfile_change', url = reverse("admin:experiments_cachedfile_change", args=(obj.cache.pk,))
args=(obj.cache.pk,))
text = obj.cache.hash text = obj.cache.hash
what = 'Cached File' what = "Cached File"
return mark_safe('%s: <a href="%s">%s</a>' % (what, url, text)) return mark_safe('%s: <a href="%s">%s</a>' % (what, url, text)) # nosec
def has_delete_permission(self, request, obj=None): def has_delete_permission(self, request, obj=None):
return False return False
def has_add_permission(self, request): def has_add_permission(self, request):
return False return False
class CachedFileInline(admin.TabularInline): class CachedFileInline(admin.TabularInline):
model = CachedFileModel.blocks.through model = CachedFileModel.blocks.through
verbose_name = 'Output' verbose_name = "Output"
verbose_name_plural = 'Outputs' verbose_name_plural = "Outputs"
extra = 0 extra = 0
readonly_fields = ['output'] readonly_fields = ["output"]
fields = readonly_fields fields = readonly_fields
def output(self, obj): def output(self, obj):
url = reverse('admin:experiments_cachedfile_change', args=(obj.cachedfile.pk,)) url = reverse("admin:experiments_cachedfile_change", args=(obj.cachedfile.pk,))
text = obj.cachedfile.hash text = obj.cachedfile.hash
what = 'Cached File' what = "Cached File"
return mark_safe('%s: <a href="%s">%s</a>' % (what, url, text)) return mark_safe('%s: <a href="%s">%s</a>' % (what, url, text)) # nosec
def has_delete_permission(self, request, obj=None): def has_delete_permission(self, request, obj=None):
return False return False
def has_add_permission(self, request): def has_add_permission(self, request):
return False return False
class BlockDependentsInline(admin.TabularInline): class BlockDependentsInline(admin.TabularInline):
model = BlockModel.dependencies.through model = BlockModel.dependencies.through
verbose_name = 'Dependent' verbose_name = "Dependent"
verbose_name_plural = 'Dependents' verbose_name_plural = "Dependents"
fk_name = 'to_block' fk_name = "to_block"
extra = 0 extra = 0
readonly_fields = ['order', 'name', 'algorithm', 'analyzer', readonly_fields = ["order", "name", "algorithm", "analyzer", "status"]
'status'] ordering = ["id"]
ordering = ['id']
fields = readonly_fields fields = readonly_fields
def order(self, obj): def order(self, obj):
return obj.from_block.execution_order return obj.from_block.execution_order
def name(self, obj): def name(self, obj):
url = reverse('admin:experiments_block_change', args=(obj.from_block.pk,)) url = reverse("admin:experiments_block_change", args=(obj.from_block.pk,))
return mark_safe('<a href="%s">%s</a>' % (url, obj.from_block.name)) return mark_safe('<a href="%s">%s</a>' % (url, obj.from_block.name)) # nosec
def algorithm(self, obj): def algorithm(self, obj):
return obj.from_block.algorithm return obj.from_block.algorithm
def analyzer(self, obj): def analyzer(self, obj):
return obj.from_block.analyzer return obj.from_block.analyzer
analyzer.boolean = True analyzer.boolean = True
def status(self, obj): def status(self, obj):
...@@ -329,33 +329,34 @@ class BlockDependentsInline(admin.TabularInline): ...@@ -329,33 +329,34 @@ class BlockDependentsInline(admin.TabularInline):
return False return False
def has_add_permission(self, request): def has_add_permission(self, request):
return False return False
class BlockDependenciesInline(admin.TabularInline): class BlockDependenciesInline(admin.TabularInline):
model = BlockModel.dependencies.through model = BlockModel.dependencies.through
verbose_name = 'Dependency' verbose_name = "Dependency"
verbose_name_plural = 'Dependencies' verbose_name_plural = "Dependencies"
fk_name = 'from_block' fk_name = "from_block"
extra = 0 extra = 0
readonly_fields = ['order', 'name', 'algorithm', 'analyzer', 'status'] readonly_fields = ["order", "name", "algorithm", "analyzer", "status"]
ordering = ['id'] ordering = ["id"]
fields = readonly_fields fields = readonly_fields
def order(self, obj): def order(self, obj):
return obj.to_block.execution_order return obj.to_block.execution_order
def name(self, obj): def name(self, obj):
url = reverse('admin:experiments_block_change', args=(obj.to_block.pk,)) url = reverse("admin:experiments_block_change", args=(obj.to_block.pk,))
return mark_safe('<a href="%s">%s</a>' % (url, obj.to_block.name)) return mark_safe('<a href="%s">%s</a>' % (url, obj.to_block.name)) # nosec
def algorithm(self, obj): def algorithm(self, obj):
return obj.to_block.algorithm return obj.to_block.algorithm
def analyzer(self, obj): def analyzer(self, obj):
return obj.to_block.analyzer return obj.to_block.analyzer
analyzer.boolean = True analyzer.boolean = True
def status(self, obj): def status(self, obj):
...@@ -365,15 +366,12 @@ class BlockDependenciesInline(admin.TabularInline): ...@@ -365,15 +366,12 @@ class BlockDependenciesInline(admin.TabularInline):
return False return False
def has_add_permission(self, request): def has_add_permission(self, request):
return False return False
class BlockModelForm(forms.ModelForm): class BlockModelForm(forms.ModelForm):
command = CodeMirrorJSONCharField( command = CodeMirrorJSONCharField(help_text=Messages["json"], readonly=True,)
help_text=Messages['json'],
readonly=True,
)
class Meta: class Meta:
model = BlockModel model = BlockModel
...@@ -383,34 +381,34 @@ class BlockModelForm(forms.ModelForm): ...@@ -383,34 +381,34 @@ class BlockModelForm(forms.ModelForm):
class Block(admin.ModelAdmin): class Block(admin.ModelAdmin):
list_display = ( list_display = (
'id', "id",
'author', "author",
'toolchain', "toolchain",
'xp', "xp",
'execution_order', "execution_order",
'name', "name",
'algorithm', "algorithm",
'analyzer', "analyzer",
'status', "status",
'ins', "ins",
'outs', "outs",
'environment', "environment",
'q' "q",
) )
search_fields = [ search_fields = [
'name', "name",
'experiment__author__username', "experiment__author__username",
'experiment__toolchain__author__username', "experiment__toolchain__author__username",
'experiment__toolchain__name', "experiment__toolchain__name",
'experiment__name', "experiment__name",
'algorithm__author__username', "algorithm__author__username",
'algorithm__name', "algorithm__name",
'environment__name', "environment__name",
'environment__version', "environment__version",
] ]
list_display_links = ('id', 'name') list_display_links = ("id", "name")
inlines = [ inlines = [
BlockDependenciesInline, BlockDependenciesInline,
...@@ -419,11 +417,11 @@ class Block(admin.ModelAdmin): ...@@ -419,11 +417,11 @@ class Block(admin.ModelAdmin):
BlockDependentsInline, BlockDependentsInline,
] ]
exclude = ['dependencies'] exclude = ["dependencies"]
def get_queryset(self, request): def get_queryset(self, request):
qs = super(Block, self).get_queryset(request) qs = super(Block, self).get_queryset(request)
return qs.annotate(Count('outputs')) return qs.annotate(Count("outputs"))
def author(self, obj): def author(self, obj):
return obj.experiment.author return obj.experiment.author
...@@ -433,116 +431,106 @@ class Block(admin.ModelAdmin): ...@@ -433,116 +431,106 @@ class Block(admin.ModelAdmin):
def xp(self, obj): def xp(self, obj):
return obj.experiment.name return obj.experiment.name
xp.short_description = 'experiment'
xp.short_description = "experiment"
def ins(self, obj): def ins(self, obj):
return obj.inputs.count() return obj.inputs.count()
def outs(self, obj): def outs(self, obj):
return obj.outputs__count return obj.outputs__count
outs.admin_order_field = 'outputs__count'
outs.admin_order_field = "outputs__count"
def q(self, obj): def q(self, obj):
if obj.queue: return obj.queue.name if obj.queue:
return obj.queue.name
return None return None
q.short_description = 'queue'
q.short_description = "queue"
def get_readonly_fields(self, request, obj=None): def get_readonly_fields(self, request, obj=None):
return list(self.readonly_fields) + \ return list(self.readonly_fields) + [
[field.name for field in obj._meta.fields if field.name != 'command'] field.name for field in obj._meta.fields if field.name != "command"
]
def has_delete_permission(self, request, obj=None): def has_delete_permission(self, request, obj=None):
return False return False
def has_add_permission(self, request): def has_add_permission(self, request):
return False return False
form = BlockModelForm form = BlockModelForm
fieldsets = ( fieldsets = (
(None, (None, dict(fields=("id", "name", "experiment"),),),
dict( (
fields=('id', 'name', 'experiment'), "Status and dates",
), dict(fields=("creation_date", "start_date", "end_date", "status"),),
), ),
('Status and dates', ("Code", dict(classes=("collapse",), fields=("algorithm", "analyzer",),),),
dict( (
fields=('creation_date', 'start_date', 'end_date', 'status'), "Backend",
), dict(
), classes=("collapse",),
('Code', fields=("environment", "queue", "required_slots", "channel"),
dict( ),
classes=('collapse',), ),
fields=('algorithm', 'analyzer',), ("Command", dict(classes=("collapse",), fields=("command",),),),
),
),
('Backend',
dict(
classes=('collapse',),
fields=('environment', 'queue', 'required_slots', 'channel'),
),
),
('Command',
dict(
classes=('collapse',),
fields=('command',),
),
),
) )
admin.site.register(BlockModel, Block) admin.site.register(BlockModel, Block)
#---------------------------------------------------------- # ----------------------------------------------------------
class Result(admin.ModelAdmin): class Result(admin.ModelAdmin):
list_display = ('id', 'cache', 'name', 'type', 'primary', 'data_value') list_display = ("id", "cache", "name", "type", "primary", "data_value")
search_fields = [ search_fields = [
'name', "name",
'cache__hash', "cache__hash",
] ]
list_display_links = ('id', 'name') list_display_links = ("id", "name")
list_select_related = ( list_select_related = ("cache",)
'cache',
)
def get_readonly_fields(self, request, obj=None): def get_readonly_fields(self, request, obj=None):
return list(self.readonly_fields) + \ return list(self.readonly_fields) + [field.name for field in obj._meta.fields]
[field.name for field in obj._meta.fields]
def has_delete_permission(self, request, obj=None): def has_delete_permission(self, request, obj=None):
return False return False
def has_add_permission(self, request): def has_add_permission(self, request):
return False return False
admin.site.register(ResultModel, Result) admin.site.register(ResultModel, Result)
#---------------------------------------------------------- # ----------------------------------------------------------
def delete_file_on_fs(modeladmin, request, queryset): def delete_file_on_fs(modeladmin, request, queryset):
''' """
Delete the files contained in the cache Delete the files contained in the cache
''' """
for obj in queryset: for obj in queryset:
obj.delete_files() obj.delete_files()
delete_file_on_fs.short_description = 'Delete files from the cache' delete_file_on_fs.short_description = "Delete files from the cache"
def cascading_delete_file_on_fs(modeladmin, request, queryset): def cascading_delete_file_on_fs(modeladmin, request, queryset):
''' """
Delete the files contained in the cache Delete the files contained in the cache
''' """
for obj in queryset: for obj in queryset:
for block in obj.blocks.all(): for block in obj.blocks.all():
...@@ -556,108 +544,104 @@ def cascading_delete_file_on_fs(modeladmin, request, queryset): ...@@ -556,108 +544,104 @@ def cascading_delete_file_on_fs(modeladmin, request, queryset):
input_.cache.delete_files() input_.cache.delete_files()
cascading_delete_file_on_fs.short_description = 'Delete files from the ' \ cascading_delete_file_on_fs.short_description = (
'selected and related caches' "Delete files from the " "selected and related caches"
)
class CachedFile(admin.ModelAdmin): class CachedFile(admin.ModelAdmin):
search_fields = [ search_fields = [
'hash', "hash",
'blocks__name', "blocks__name",
'blocks__experiment__name', "blocks__experiment__name",
] ]
list_display = ( list_display = (
'id', "id",
'hash', "hash",
'status', "status",
'date', "date",
'blocks_url', "blocks_url",
) )
list_display_links = ('id', 'hash') list_display_links = ("id", "hash")
list_filter = ('status', ) list_filter = ("status",)
# to avoid very slow loading of cached files # to avoid very slow loading of cached files
raw_id_fields = ('blocks',) raw_id_fields = ("blocks",)
actions = [delete_file_on_fs, cascading_delete_file_on_fs] actions = [delete_file_on_fs, cascading_delete_file_on_fs]
def get_queryset(self, request): def get_queryset(self, request):
qs = super(CachedFile, self).get_queryset(request) qs = super(CachedFile, self).get_queryset(request)
return qs.annotate(date=Max('blocks__start_date')) return qs.annotate(date=Max("blocks__start_date"))
def get_actions(self, request): def get_actions(self, request):
actions = super(CachedFile, self).get_actions(request) actions = super(CachedFile, self).get_actions(request)
if 'delete_selected' in actions: if "delete_selected" in actions:
del actions['delete_selected'] del actions["delete_selected"]
return actions return actions
def date(self, obj): def date(self, obj):
return obj.date return obj.date
date.admin_order_field = '-date' date.admin_order_field = "-date"
def blocks_url(self, obj): def blocks_url(self, obj):
retval = '<ul>' retval = "<ul>"
for block in obj.blocks.all(): for block in obj.blocks.all():
retval += format_html("<li><a href='{block_url}'>{block_name}</a> @ <a href='{experiment_url}'>{experiment_name}</a> ({block_status})</li>", retval += format_html(
experiment_url=reverse('admin:experiments_experiment_change', args=(block.experiment.id,)), "<li><a href='{block_url}'>{block_name}</a> @ <a href='{experiment_url}'>{experiment_name}</a> ({block_status})</li>",
experiment_name=block.experiment.fullname(), experiment_url=reverse(
block_url=reverse('admin:experiments_block_change', args=(block.id,)), "admin:experiments_experiment_change", args=(block.experiment.id,)
block_name=block.name, ),
block_status=block.get_status_display(), experiment_name=block.experiment.fullname(),
block_url=reverse("admin:experiments_block_change", args=(block.id,)),
block_name=block.name,
block_status=block.get_status_display(),
) )
return retval + '</ul>' return retval + "</ul>"
blocks_url.short_description = "Blocks" blocks_url.short_description = "Blocks"
blocks_url.allow_tags = True blocks_url.allow_tags = True
fieldsets = ( fieldsets = (
(None, (None, dict(fields=("hash", "status", "blocks",)),),
dict( ("Logging", dict(fields=("error_report", "stderr", "stdout"),),),
fields=('hash', 'status', 'blocks',) (
), "Performance",
), dict(
('Logging', classes=("collapse",),
dict( fields=(
fields=('error_report', 'stderr', 'stdout'), "linear_execution_time",
), "speed_up_real",
), "speed_up_maximal",
('Performance', "cpu_time",
dict( "max_memory",
classes=('collapse',), "queuing_time",
fields=( "data_read_time",
'linear_execution_time', "data_read_size",
'speed_up_real', "data_read_nb_blocks",
'speed_up_maximal', "data_written_time",
'cpu_time', "data_written_size",
'max_memory', "data_written_nb_blocks",
'queuing_time', ),
'data_read_time', ),
'data_read_size', ),
'data_read_nb_blocks',
'data_written_time',
'data_written_size',
'data_written_nb_blocks',
),
),
),
) )
readonly_fields = ['blocks'] readonly_fields = ["blocks"]
def get_readonly_fields(self, request, obj=None): def get_readonly_fields(self, request, obj=None):
return list(self.readonly_fields) + \ return list(self.readonly_fields) + [field.name for field in obj._meta.fields]
[field.name for field in obj._meta.fields]
def has_delete_permission(self, request, obj=None): def has_delete_permission(self, request, obj=None):
return False return False
def has_add_permission(self, request): def has_add_permission(self, request):
return False return False
admin.site.register(CachedFileModel, CachedFile) admin.site.register(CachedFileModel, CachedFile)
beat/web/experiments/api.py
+ 22
− 28
View file @ 0e7d7bb4
...@@ -25,48 +25,42 @@ ...@@ -25,48 +25,42 @@
# # # #
############################################################################### ###############################################################################
import simplejson
import functools import functools
import simplejson
from django.conf import settings from django.conf import settings
from django.shortcuts import get_object_or_404
from django.core.exceptions import ValidationError from django.core.exceptions import ValidationError
from django.shortcuts import get_object_or_404
from rest_framework.response import Response
from rest_framework import permissions
from rest_framework import generics from rest_framework import generics
from rest_framework import permissions
from rest_framework import serializers from rest_framework import serializers
from rest_framework.views import APIView
from rest_framework.reverse import reverse
from rest_framework.exceptions import ParseError from rest_framework.exceptions import ParseError
from rest_framework.response import Response
from rest_framework.reverse import reverse
from rest_framework.views import APIView
import beat.core.hash
import beat.core.algorithm import beat.core.algorithm
import beat.core.hash
import beat.core.toolchain import beat.core.toolchain
from .models import Experiment from ..common.api import ListContributionView
from .serializers import ( from ..common.api import ListCreateContributionView
ExperimentSerializer, from ..common.api import RetrieveUpdateDestroyContributionView
ExperimentResultsSerializer, from ..common.api import ShareView
ExperimentCreationSerializer,
)
from .permissions import IsDatabaseAccessible
from ..common.responses import BadRequestResponse, ForbiddenResponse
from ..common.api import (
ShareView,
ListContributionView,
ListCreateContributionView,
RetrieveUpdateDestroyContributionView,
)
from ..common.mixins import CommonContextMixin
from ..common.exceptions import ShareError from ..common.exceptions import ShareError
from ..common.mixins import CommonContextMixin
from ..common.responses import BadRequestResponse
from ..common.responses import ForbiddenResponse
from ..common.serializers import SharingSerializer from ..common.serializers import SharingSerializer
from ..common.utils import validate_restructuredtext, ensure_html, py3_cmp from ..common.utils import ensure_html
from ..common.utils import py3_cmp
from ..common.utils import validate_restructuredtext
from ..toolchains.models import Toolchain from ..toolchains.models import Toolchain
from .models import Experiment
from .permissions import IsDatabaseAccessible
from .serializers import ExperimentCreationSerializer
from .serializers import ExperimentResultsSerializer
from .serializers import ExperimentSerializer
# ---------------------------------------------------------- # ----------------------------------------------------------
......
beat/web/experiments/apps.py
+ 9
− 5
View file @ 0e7d7bb4
...@@ -25,15 +25,19 @@ ...@@ -25,15 +25,19 @@
# # # #
############################################################################### ###############################################################################
from ..common.apps import CommonAppConfig
from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext_lazy as _
from ..common.apps import CommonAppConfig
class ExperimentsConfig(CommonAppConfig): class ExperimentsConfig(CommonAppConfig):
name = 'beat.web.experiments' name = "beat.web.experiments"
verbose_name = _('Experiments') verbose_name = _("Experiments")
def ready(self): def ready(self):
super(ExperimentsConfig, self).ready() super(ExperimentsConfig, self).ready()
from .signals import on_team_delete
from actstream import registry from actstream import registry
registry.register(self.get_model('Experiment'))
from .signals import on_team_delete # noqa: F401
registry.register(self.get_model("Experiment"))
beat/web/experiments/serializers.py
+ 9
− 16
View file @ 0e7d7bb4
...@@ -25,31 +25,24 @@ ...@@ -25,31 +25,24 @@
# # # #
############################################################################### ###############################################################################
import simplejson as json
import beat.core
from datetime import datetime from datetime import datetime
from rest_framework import serializers import simplejson as json
from django.contrib.humanize.templatetags.humanize import naturaltime
from rest_framework import exceptions as drf_exceptions from rest_framework import exceptions as drf_exceptions
from rest_framework import serializers
from django.contrib.humanize.templatetags.humanize import naturaltime import beat.core
from ..common import fields as beat_fields
from ..common.serializers import ShareableSerializer from ..common.serializers import ShareableSerializer
from ..common.utils import validate_restructuredtext
from ..common.utils import annotate_full_name from ..common.utils import annotate_full_name
from ..common import fields as beat_fields from ..common.utils import validate_restructuredtext
from ..ui.templatetags.markup import restructuredtext
from ..toolchains.models import Toolchain from ..toolchains.models import Toolchain
from ..ui.templatetags.markup import restructuredtext
from .models.experiment import validate_experiment
from .models import Experiment
from .models import Block from .models import Block
from .models import Experiment
from .models.experiment import validate_experiment
# ---------------------------------------------------------- # ----------------------------------------------------------
......
beat/web/experiments/signals.py
+ 8
− 6
View file @ 0e7d7bb4
...@@ -25,13 +25,12 @@ ...@@ -25,13 +25,12 @@
# # # #
############################################################################### ###############################################################################
from django.db import models from django.db import models
from django.dispatch import receiver from django.dispatch import receiver
from ..team.models import Team from ..team.models import Team
from .models import Experiment, Block from .models import Experiment
from datetime import datetime
# These two auto-delete files from filesystem when they are unneeded: # These two auto-delete files from filesystem when they are unneeded:
...@@ -70,7 +69,8 @@ def auto_delete_file_on_change(sender, instance, **kwargs): ...@@ -70,7 +69,8 @@ def auto_delete_file_on_change(sender, instance, **kwargs):
old_descr.delete(save=False) old_descr.delete(save=False)
#_________ Algorithms _________ # _________ Algorithms _________
def build_user_algorithm_set(user): def build_user_algorithm_set(user):
all_algorithms = [] all_algorithms = []
...@@ -80,6 +80,7 @@ def build_user_algorithm_set(user): ...@@ -80,6 +80,7 @@ def build_user_algorithm_set(user):
return set(all_algorithms) return set(all_algorithms)
def process_algorithms(team): def process_algorithms(team):
team_algorithms = set(team.shared_algorithms.all() | team.usable_algorithms.all()) team_algorithms = set(team.shared_algorithms.all() | team.usable_algorithms.all())
...@@ -90,7 +91,8 @@ def process_algorithms(team): ...@@ -90,7 +91,8 @@ def process_algorithms(team):
algorithm.share(public=False, users=[member]) algorithm.share(public=False, users=[member])
#_________ Toolchains _________ # _________ Toolchains _________
def build_user_toolchain_set(user): def build_user_toolchain_set(user):
all_toolchains = [] all_toolchains = []
...@@ -113,6 +115,6 @@ def process_toolchains(team): ...@@ -113,6 +115,6 @@ def process_toolchains(team):
@receiver(models.signals.pre_delete, sender=Team) @receiver(models.signals.pre_delete, sender=Team)
def on_team_delete(sender, **kwargs): def on_team_delete(sender, **kwargs):
team = kwargs.get('instance') team = kwargs.get("instance")
process_algorithms(team) process_algorithms(team)
process_toolchains(team) process_toolchains(team)
beat/web/experiments/utils.py
+ 5
− 6
View file @ 0e7d7bb4
...@@ -26,26 +26,25 @@ ...@@ -26,26 +26,25 @@
############################################################################### ###############################################################################
'''Utilities for experiment management''' """Utilities for experiment management"""
import logging
from django.db.models import Count from django.db.models import Count
from .models import CachedFile from .models import CachedFile
import logging
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
def list_orphaned_cachedfiles(): def list_orphaned_cachedfiles():
'''Lists orphaned cache files that do not exist in the disk either''' """Lists orphaned cache files that do not exist in the disk either"""
q = CachedFile.objects.annotate(Count('blocks')).filter(blocks__count__lt=1) q = CachedFile.objects.annotate(Count("blocks")).filter(blocks__count__lt=1)
return [c for c in q if not c.exists()] return [c for c in q if not c.exists()]
def cleanup_orphaned_cachedfiles(): def cleanup_orphaned_cachedfiles():
'''Cleans-up orphaned cache files that do not exist in the disk either''' """Cleans-up orphaned cache files that do not exist in the disk either"""
for c in list_orphaned_cachedfiles(): for c in list_orphaned_cachedfiles():
logger.info("Removing orphaned CachedFile object `%s'..." % c.hash) logger.info("Removing orphaned CachedFile object `%s'..." % c.hash)
......
beat/web/experiments/views.py
+ 101
− 84
View file @ 0e7d7bb4
...@@ -25,26 +25,27 @@ ...@@ -25,26 +25,27 @@
# # # #
############################################################################### ###############################################################################
from django.shortcuts import get_object_or_404 from django.conf import settings
from django.shortcuts import render, redirect
from django.http import Http404
from django.contrib.auth.decorators import login_required from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User from django.contrib.auth.models import User
from django.conf import settings
from django.db.models.functions import Coalesce from django.db.models.functions import Coalesce
from django.http import Http404
from django.shortcuts import get_object_or_404
from django.shortcuts import redirect
from django.shortcuts import render
from .models import Experiment
from ..toolchains.models import Toolchain
from ..team.models import Team from ..team.models import Team
from ..toolchains.models import Toolchain
from .models import Experiment
# ----------------------------------------------------------
#----------------------------------------------------------
@login_required @login_required
def new_from_toolchain(request, toolchain_author_name, toolchain_name, def new_from_toolchain(
toolchain_version): request, toolchain_author_name, toolchain_name, toolchain_version
'''Sets up a new experiment from a toolchain name''' ):
"""Sets up a new experiment from a toolchain name"""
# Retrieve the toolchain # Retrieve the toolchain
toolchain = get_object_or_404( toolchain = get_object_or_404(
...@@ -56,23 +57,22 @@ def new_from_toolchain(request, toolchain_author_name, toolchain_name, ...@@ -56,23 +57,22 @@ def new_from_toolchain(request, toolchain_author_name, toolchain_name,
# Check that the user can access it # Check that the user can access it
has_access = toolchain.accessibility_for(request.user)[0] has_access = toolchain.accessibility_for(request.user)[0]
if not(has_access): raise Http404() if not (has_access):
raise Http404()
return render(request, return render(
'experiments/setup.html', request, "experiments/setup.html", {"toolchain": toolchain, "action": "new"}
{ )
'toolchain': toolchain,
'action': 'new',
})
#---------------------------------------------------------- # ----------------------------------------------------------
@login_required @login_required
def fork(request, author_name, toolchain_author_name, def fork(
toolchain_name, toolchain_version, name): request, author_name, toolchain_author_name, toolchain_name, toolchain_version, name
'''Sets up a new experiment from an experiment fork''' ):
"""Sets up a new experiment from an experiment fork"""
# Retrieve the experiment # Retrieve the experiment
experiment = get_object_or_404( experiment = get_object_or_404(
...@@ -81,28 +81,31 @@ def fork(request, author_name, toolchain_author_name, ...@@ -81,28 +81,31 @@ def fork(request, author_name, toolchain_author_name,
toolchain__author__username=toolchain_author_name, toolchain__author__username=toolchain_author_name,
toolchain__name=toolchain_name, toolchain__name=toolchain_name,
toolchain__version=toolchain_version, toolchain__version=toolchain_version,
name=name name=name,
) )
# Check that the user can access it # Check that the user can access it
(has_access, accessibility) = experiment.accessibility_for(request.user) (has_access, accessibility) = experiment.accessibility_for(request.user)
if not(has_access): raise Http404() if not (has_access):
raise Http404()
return render(
request,
"experiments/setup.html",
{
"toolchain": experiment.toolchain,
"experiment": experiment,
"action": "fork",
},
)
return render(request,
'experiments/setup.html',
{
'toolchain': experiment.toolchain,
'experiment': experiment,
'action': 'fork',
})
# ----------------------------------------------------------
#----------------------------------------------------------
@login_required @login_required
def reset(request, toolchain_author_name, def reset(request, toolchain_author_name, toolchain_name, toolchain_version, name):
toolchain_name, toolchain_version, name): """Resets the current experiment so it can run again"""
'''Resets the current experiment so it can run again'''
# Retrieve the experiment # Retrieve the experiment
experiment = get_object_or_404( experiment = get_object_or_404(
...@@ -111,19 +114,21 @@ def reset(request, toolchain_author_name, ...@@ -111,19 +114,21 @@ def reset(request, toolchain_author_name,
toolchain__author__username=toolchain_author_name, toolchain__author__username=toolchain_author_name,
toolchain__name=toolchain_name, toolchain__name=toolchain_name,
toolchain__version=toolchain_version, toolchain__version=toolchain_version,
name=name name=name,
) )
if not experiment.deletable(): raise Http404() if not experiment.deletable():
raise Http404()
experiment.reset() experiment.reset()
return redirect(experiment) return redirect(experiment)
def view(request, author_name, toolchain_author_name, toolchain_name, def view(
toolchain_version, name): request, author_name, toolchain_author_name, toolchain_name, toolchain_version, name
'''Views an experiment no matter its present state''' ):
"""Views an experiment no matter its present state"""
# Retrieve the experiment # Retrieve the experiment
experiment = get_object_or_404( experiment = get_object_or_404(
...@@ -132,80 +137,92 @@ def view(request, author_name, toolchain_author_name, toolchain_name, ...@@ -132,80 +137,92 @@ def view(request, author_name, toolchain_author_name, toolchain_name,
toolchain__author__username=toolchain_author_name, toolchain__author__username=toolchain_author_name,
toolchain__name=toolchain_name, toolchain__name=toolchain_name,
toolchain__version=toolchain_version, toolchain__version=toolchain_version,
name=name name=name,
) )
# Check that the user can access it # Check that the user can access it
(has_access, accessibility) = experiment.accessibility_for(request.user) (has_access, accessibility) = experiment.accessibility_for(request.user)
if not(has_access): raise Http404() if not (has_access):
raise Http404()
if experiment.status == Experiment.PENDING: if experiment.status == Experiment.PENDING:
if request.user.is_anonymous: raise Http404() if request.user.is_anonymous:
return render(request, raise Http404()
'experiments/setup.html', return render(
{ request,
'toolchain': experiment.toolchain, "experiments/setup.html",
'experiment': experiment, {
'action': 'pending', "toolchain": experiment.toolchain,
}) "experiment": experiment,
"action": "pending",
},
)
# Users the object can be shared with # Users the object can be shared with
users = User.objects.exclude(username__in=settings.ACCOUNTS_TO_EXCLUDE_FROM_TEAMS).order_by('username') users = User.objects.exclude(
username__in=settings.ACCOUNTS_TO_EXCLUDE_FROM_TEAMS
).order_by("username")
# The experiment was already done, show results # The experiment was already done, show results
return render(request, return render(
'experiments/view.html', request,
{ "experiments/view.html",
'experiment': experiment, {
'owner': experiment.author == request.user, "experiment": experiment,
'users': users, "owner": experiment.author == request.user,
'teams': Team.objects.for_user(request.user, True) "users": users,
}) "teams": Team.objects.for_user(request.user, True),
},
)
#---------------------------------------------------------- # ----------------------------------------------------------
def ls(request, author_name): def ls(request, author_name):
'''List all accessible experiments to the request user''' """List all accessible experiments to the request user"""
if not author_name: return public_ls(request) if not author_name:
return public_ls(request)
# check that the user exists on the system # check that the user exists on the system
author = get_object_or_404(User, username=author_name) author = get_object_or_404(User, username=author_name)
# orders so that experiments that the latest information is displayed first # orders so that experiments that the latest information is displayed first
objects = Experiment.objects.from_author_and_public(request.user, objects = (
author_name).annotate(updated=Coalesce('end_date', 'start_date', Experiment.objects.from_author_and_public(request.user, author_name)
'creation_date')).order_by('-updated') .annotate(updated=Coalesce("end_date", "start_date", "creation_date"))
.order_by("-updated")
)
if request.user.is_anonymous: if request.user.is_anonymous:
objects = objects.exclude(status=Experiment.PENDING) objects = objects.exclude(status=Experiment.PENDING)
owner = (request.user == author) owner = request.user == author
return render(request, return render(
'experiments/list.html', request,
dict( "experiments/list.html",
objects=objects, dict(objects=objects, author=author, owner=owner,),
author=author, )
owner=owner,
))
#---------------------------------------------------------- # ----------------------------------------------------------
def public_ls(request): def public_ls(request):
'''List all publicly accessible experiments''' """List all publicly accessible experiments"""
# orders so that recent objects are displayed first # orders so that recent objects are displayed first
objects = Experiment.objects.public().exclude(status=Experiment.PENDING).annotate(updated=Coalesce('end_date', 'start_date', 'creation_date')).order_by('-updated') objects = (
Experiment.objects.public()
return render(request, .exclude(status=Experiment.PENDING)
'experiments/list.html', .annotate(updated=Coalesce("end_date", "start_date", "creation_date"))
dict( .order_by("-updated")
objects=objects, )
author=request.user, #anonymous
owner=False, return render(
)) request,
"experiments/list.html",
dict(objects=objects, author=request.user, owner=False,), # anonymous
)
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment