Skip to content
Snippets Groups Projects
Commit db4e9968 authored by André Anjos's avatar André Anjos :speech_balloon:
Browse files

[databases] Complete model to include dataset output and hashes

parent a53737de
No related branches found
No related tags found
1 merge request!194Scheduler
Hide whitespace changes
Inline Side-by-side
beat/web/databases/admin.py
+ 16
− 4
View file @ db4e9968
......@@ -33,8 +33,9 @@ from django.utils import six
from .models import Database as DatabaseModel
from .models import DatabaseProtocol as DatabaseProtocolModel
from .models import DatabaseSet as DatabaseSetModel
from .models import DatabaseSetOutput as DatabaseSetOutputModel
from .models import DatabaseSetTemplate as DatabaseSetTemplateModel
from .models import DatabaseOutput as DatabaseOutputModel
from .models import DatabaseSetTemplateOutput as DatabaseSetTemplateOutputModel
from .models import validate_database
from ..ui.forms import CodeMirrorJSONFileField, CodeMirrorRSTFileField, \
......@@ -234,9 +235,9 @@ admin.site.register(DatabaseModel, Database)
#------------------------------------------------
class DatabaseOutputInline(admin.TabularInline):
class DatabaseSetTemplateOutputInline(admin.TabularInline):
model = DatabaseOutputModel
model = DatabaseSetTemplateOutputModel
extra = 0
ordering = ('name',)
......@@ -248,7 +249,7 @@ class DatabaseSetTemplate(admin.ModelAdmin):
list_display_links = ('id', 'name')
inlines = [
DatabaseOutputInline,
DatabaseSetTemplateOutputInline,
]
admin.site.register(DatabaseSetTemplateModel, DatabaseSetTemplate)
......@@ -257,6 +258,13 @@ admin.site.register(DatabaseSetTemplateModel, DatabaseSetTemplate)
#------------------------------------------------
class DatabaseSetOutputInline(admin.TabularInline):
model = DatabaseSetOutputModel
extra = 0
ordering = ('hash',)
class DatabaseSet(admin.ModelAdmin):
list_display = ('id', 'protocol', 'name', 'template')
......@@ -268,4 +276,8 @@ class DatabaseSet(admin.ModelAdmin):
'protocol__name']
list_display_links = ('id', 'name')
inlines = [
DatabaseSetOutputInline,
]
admin.site.register(DatabaseSetModel, DatabaseSet)
beat/web/databases/models.py
+ 42
− 206
View file @ db4e9968
......@@ -263,6 +263,8 @@ class Database(Versionable):
result.extend(database_protocol.all_needed_dataformats())
return list(set(result))
def core(self):
return validate_database(self.declaration)
#_____ Properties __________
......@@ -275,173 +277,6 @@ class Database(Versionable):
#----------------------------------------------------------
@receiver(models.signals.pre_delete, sender=Database)
def delete_protocols(sender, **kwargs):
instance = kwargs['instance']
instance.protocols.all().delete()
#----------------------------------------------------------
# These two auto-delete files from filesystem when they are unneeded:
@receiver(models.signals.post_delete, sender=Database)
def auto_delete_file_on_delete(sender, instance, **kwargs):
"""Deletes file from filesystem when ``Database`` object is deleted.
"""
if instance.declaration_file:
instance.declaration_file.delete(save=False)
if instance.source_code_file:
instance.source_code_file.delete(save=False)
if instance.description_file:
instance.description_file.delete(save=False)
@receiver(models.signals.pre_save, sender=Database)
def auto_delete_file_on_change(sender, instance, **kwargs):
"""Deletes file from filesystem when ``Database`` object is changed."""
if not instance.pk:
return False
try:
old_file = Database.objects.get(pk=instance.pk).declaration_file
except Database.DoesNotExist:
return False
if old_file != instance.declaration_file:
old_file.delete(save=False)
try:
old_code = Database.objects.get(pk=instance.pk).source_code_file
except Database.DoesNotExist:
return False
if old_code != instance.source_code_file:
old_code.delete(save=False)
try:
old_descr = Database.objects.get(pk=instance.pk).description_file
except Database.DoesNotExist:
return False
if old_descr != instance.description_file:
old_descr.delete(save=False)
#----------------------------------------------------------
@receiver(models.signals.post_save, sender=Database)
def refresh_protocols(sender, instance, **kwargs):
"""Refreshes changed protocols"""
try:
json_declaration = instance.declaration
protocols = DatabaseProtocol.objects.filter(
database__name=instance.name,
database__version=instance.version,
)
existing = set((k.name, k.set_template_basename()) for k in protocols)
new_objects = set((k['name'], k['template']) for k in json_declaration['protocols'])
for protocol_name, template in existing - new_objects:
# notice: no need to worry, this will clean-up all the rest
protocols.get(name__iexact=protocol_name).delete()
json_protocols = dict([(k['name'], k) for k in json_declaration['protocols']])
for protocol_name, template in new_objects - existing:
protocol = DatabaseProtocol(name=protocol_name, database=instance)
protocol.save()
json_protocol = json_protocols[protocol_name]
# creates all the template sets, outputs, etc for the first time
for set_attr in json_protocol['sets']:
tset_name = json_protocol['template'] + '__' + set_attr['template']
dataset_template = DatabaseSetTemplate.objects.filter(name=tset_name)
if not dataset_template: #create
dataset_template = DatabaseSetTemplate(name=tset_name)
dataset_template.save()
else:
dataset_template = dataset_template[0]
# Create the databaset
dataset_set = DatabaseSet.objects.filter(
name = set_attr['name'],
template = dataset_template,
protocol = protocol,
)
if not dataset_set: #create
dataset_set = DatabaseSet(
name = set_attr['name'],
template = dataset_template,
protocol = protocol,
)
dataset_set.save()
# Create the database set output
for output_name, format_name in set_attr['outputs'].items():
if len(format_name.split('/')) != 3:
raise SyntaxError(
"Dataformat should be named following the style " \
"`<user>/<format>/<version>', the " \
"value `%s' is not valid" % (
format_name,
)
)
(author, name, version) = format_name.split('/')
dataformats = DataFormat.objects.filter(
author__username=author,
name=name,
version=version,
)
# TODO: Remove this when validation works (see comments)
if len(dataformats) != 1:
raise SyntaxError(
"Could not find dataformat named `%s' to set" \
"output `%s' of template `%s' for protocol" \
"`%s' of database `%s'", (
format_name,
output_name,
dataset_template.name,
protocol_name,
instance.name,
)
)
return
database_output = DatabaseOutput.objects.filter(
name=output_name,
template=dataset_template,
dataformat=dataformats[0],
)
if not database_output: # create
database_output = DatabaseOutput(
name=output_name,
template=dataset_template,
dataformat=dataformats[0],
)
database_output.save()
except Exception:
instance.delete() #do we need this or is it auto-rolled back?
raise
#----------------------------------------------------------
class DatabaseProtocolManager(models.Manager):
def get_by_natural_key(self, database_name, database_version, name):
......@@ -454,10 +289,11 @@ class DatabaseProtocolManager(models.Manager):
class DatabaseProtocol(models.Model):
objects = DatabaseProtocolManager()
objects = DatabaseProtocolManager()
database = models.ForeignKey(Database, related_name='protocols')
name = models.CharField(max_length=200, blank=True)
database = models.ForeignKey(Database, related_name='protocols',
on_delete=models.CASCADE)
name = models.CharField(max_length=200, blank=True)
class Meta:
unique_together = ('database', 'name')
......@@ -496,16 +332,6 @@ class DatabaseProtocol(models.Model):
#----------------------------------------------------------
@receiver(models.signals.pre_delete, sender=DatabaseProtocol)
def delete_sets(sender, **kwargs):
instance = kwargs['instance']
instance.sets.all().delete()
#----------------------------------------------------------
class DatabaseSetTemplateManager(models.Manager):
def get_by_natural_key(self, name):
......@@ -528,15 +354,6 @@ class DatabaseSetTemplate(models.Model):
#----------------------------------------------------------
@receiver(models.signals.pre_delete, sender=DatabaseSetTemplate)
def delete_outputs(sender, **kwargs):
instance = kwargs['instance']
instance.outputs.all().delete()
#----------------------------------------------------------
class DatabaseSetManager(models.Manager):
def get_by_natural_key(self, database_name, database_version, protocol_name, name, template_name):
......@@ -553,9 +370,11 @@ class DatabaseSet(models.Model):
objects = DatabaseSetManager()
protocol = models.ForeignKey(DatabaseProtocol, related_name='sets')
protocol = models.ForeignKey(DatabaseProtocol, related_name='sets',
on_delete=models.CASCADE)
name = models.CharField(max_length=200, blank=True)
template = models.ForeignKey(DatabaseSetTemplate, related_name='sets')
template = models.ForeignKey(DatabaseSetTemplate, related_name='sets',
on_delete=models.CASCADE)
class Meta:
unique_together = ('protocol', 'name', 'template')
......@@ -587,29 +406,46 @@ class DatabaseSet(models.Model):
#----------------------------------------------------------
@receiver(models.signals.post_delete, sender=DatabaseSet)
def delete_empty_template_sets(sender, **kwargs):
class DatabaseSetTemplateOutput(models.Model):
template = models.ForeignKey(DatabaseSetTemplate,
related_name='outputs', on_delete=models.CASCADE)
name = models.CharField(max_length=200)
dataformat = models.ForeignKey(DataFormat,
related_name='database_outputs', on_delete=models.CASCADE)
class Meta:
unique_together = ('template', 'name', 'dataformat')
def __str__(self):
return self.fullname()
instance = kwargs['instance']
try:
if not instance.template.sets.all(): instance.template.delete()
except:
pass
def fullname(self):
return self.template.name + '.' + self.name
#----------------------------------------------------------
class DatabaseOutput(models.Model):
template = models.ForeignKey(DatabaseSetTemplate, related_name='outputs')
name = models.CharField(max_length=200)
dataformat = models.ForeignKey(DataFormat, related_name='database_outputs')
class Meta:
unique_together = ('template', 'name', 'dataformat')
class DatabaseSetOutput(models.Model):
template = models.ForeignKey(DatabaseSetTemplateOutput,
related_name='instances', on_delete=models.CASCADE)
set = models.ForeignKey(DatabaseSet, related_name='outputs',
on_delete=models.CASCADE)
hash = models.CharField(max_length=64, unique=True)
def __str__(self):
return self.fullname()
def fullname(self):
return self.template.name + '.' + self.name
return '%s.%s.%s.%s' % (
self.set.protocol.database.fullname(),
self.set.protocol.name,
self.set.name,
self.name,
)
def all_referenced_dataformats(self):
return self.template.all_referenced_dataformats()
def all_needed_dataformats(self):
return self.template.all_needed_dataformats()
beat/web/databases/signals.py 0 → 100644
+ 214
− 0
View file @ db4e9968
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
###############################################################################
# #
# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/ #
# Contact: beat.support@idiap.ch #
# #
# This file is part of the beat.web module of the BEAT platform. #
# #
# Commercial License Usage #
# Licensees holding valid commercial BEAT licenses may use this file in #
# accordance with the terms contained in a written agreement between you #
# and Idiap. For further information contact tto@idiap.ch #
# #
# Alternatively, this file may be used under the terms of the GNU Affero #
# Public License version 3 as published by the Free Software and appearing #
# in the file LICENSE.AGPL included in the packaging of this file. #
# The BEAT platform is distributed in the hope that it will be useful, but #
# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY #
# or FITNESS FOR A PARTICULAR PURPOSE. #
# #
# You should have received a copy of the GNU Affero Public License along #
# with the BEAT platform. If not, see http://www.gnu.org/licenses/. #
# #
###############################################################################
from django.db import models
from ..dataformats.models import DataFormat
from .models import Database, DatabaseProtocol, DatabaseSet
from .models import DatabaseSetTemplate, DatabaseSetTemplateOutput
from .models import DatabaseSetOutput
@receiver(models.signals.post_delete, sender=Database)
def auto_delete_file_on_delete(sender, instance, **kwargs):
"""Deletes file from filesystem when ``Database`` object is deleted.
"""
if instance.declaration_file:
instance.declaration_file.delete(save=False)
if instance.source_code_file:
instance.source_code_file.delete(save=False)
if instance.description_file:
instance.description_file.delete(save=False)
@receiver(models.signals.pre_save, sender=Database)
def auto_delete_file_on_change(sender, instance, **kwargs):
"""Deletes file from filesystem when ``Database`` object is changed."""
if not instance.pk:
return False
try:
old_file = Database.objects.get(pk=instance.pk).declaration_file
except Database.DoesNotExist:
return False
if old_file != instance.declaration_file:
old_file.delete(save=False)
try:
old_code = Database.objects.get(pk=instance.pk).source_code_file
except Database.DoesNotExist:
return False
if old_code != instance.source_code_file:
old_code.delete(save=False)
try:
old_descr = Database.objects.get(pk=instance.pk).description_file
except Database.DoesNotExist:
return False
if old_descr != instance.description_file:
old_descr.delete(save=False)
@receiver(models.signals.post_save, sender=Database)
def refresh_protocols(sender, instance, **kwargs):
"""Refreshes changed protocols"""
try:
core = instance.core()
protocols = DatabaseProtocol.objects.filter(
database__name=instance.name,
database__version=instance.version,
)
existing = set((k.name, k.set_template_basename()) for k in protocols)
new_objects = set((k['name'], k['template']) for k in core.protocols()])
for protocol_name, template in existing - new_objects:
# notice: no need to worry, this will clean-up all the rest
protocols.get(name__iexact=protocol_name).delete()
json_protocols = dict([(k['name'], k) for k in core.protocols()]])
for protocol_name, template in new_objects - existing:
protocol = DatabaseProtocol(name=protocol_name, database=instance)
protocol.save()
json_protocol = json_protocols[protocol_name]
# creates all the template sets, outputs, etc for the first time
for set_attr in json_protocol['sets']:
tset_name = json_protocol['template'] + '__' + set_attr['template']
dataset_template = DatabaseSetTemplate.objects.filter(name=tset_name)
if not dataset_template: #create
dataset_template = DatabaseSetTemplate(name=tset_name)
dataset_template.save()
else:
dataset_template = dataset_template[0]
# Create the database set
dataset = DatabaseSet.objects.filter(
name = set_attr['name'],
template = dataset_template,
protocol = protocol,
)
if not dataset: #create
dataset = DatabaseSet(
name = set_attr['name'],
template = dataset_template,
protocol = protocol,
)
dataset.save()
# Create the database set template output
for output_name, format_name in set_attr['outputs'].items():
if len(format_name.split('/')) != 3:
raise SyntaxError(
"Dataformat should be named following the " \
"style `<username>/<format>/<version>', the " \
"value `%s' is not valid" % (format_name,)
)
(author, name, version) = format_name.split('/')
dataformats = DataFormat.objects.filter(
author__username=author,
name=name,
version=version,
)
# TODO: Remove this when validation works (see comments)
if len(dataformats) != 1:
raise SyntaxError(
"Could not find dataformat named `%s' to set" \
"output `%s' of template `%s' for protocol" \
"`%s' of database `%s'", (
format_name,
output_name,
dataset_template.name,
protocol_name,
instance.name,
)
)
return
database_template_output = \
DatabaseSetTemplateOutput.objects.filter(
name=output_name,
template=dataset_template,
dataformat=dataformats[0],
)
if not database_template_output: # create
database_template_output = \
DatabaseSetTemplateOutput(
name=output_name,
template=dataset_template,
dataformat=dataformats[0],
)
database_template_output.save()
else:
database_template_output = \
database_template_output[0]
# Create the database template output
hash = core.hash_output(protocol.name,
dataset.name, output_name)
dataset_output = \
DatabaseSetOutput.objects.filter(hash=hash)
if not dataset_output: # create
dataset_output = DatabaseSetOutput(
template=database_template_output,
set=dataset,
hash=hash,
)
dataset_output.save()
except Exception:
instance.delete() #do we need this or is it auto-rolled back?
raise
@receiver(models.signals.post_delete, sender=DatabaseSet)
def delete_empty_template_sets(sender, **kwargs):
instance = kwargs['instance']
try:
if not instance.template.sets.all(): instance.template.delete()
except:
pass
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment