Commit fc1e747c authored by Flavio TARSETTI's avatar Flavio TARSETTI

Merge branch 'cleanup_toolchains' into 'django3_migration'

Cleanup toolchains

See merge request !366
parents b407b252 efc1e53e
Pipeline #42691 passed with stage
in 15 minutes and 10 seconds
......@@ -25,4 +25,4 @@
# #
###############################################################################
default_app_config = 'beat.web.toolchains.apps.ToolchainsConfig'
default_app_config = "beat.web.toolchains.apps.ToolchainsConfig"
......@@ -29,60 +29,50 @@ from django import forms
from django.contrib import admin
from django.core.files.base import ContentFile
from .models import Toolchain as ToolchainModel
from ..ui.forms import CodeMirrorJSONFileField, CodeMirrorRSTFileField, \
NameField
from ..common.texts import Messages
from ..ui.forms import CodeMirrorJSONFileField
from ..ui.forms import CodeMirrorRSTFileField
from ..ui.forms import NameField
from .models import Toolchain as ToolchainModel
#----------------------------------------------------------
# ----------------------------------------------------------
class ToolchainModelForm(forms.ModelForm):
name = NameField(
widget=forms.TextInput(attrs=dict(size=80)),
help_text=Messages['name'],
widget=forms.TextInput(attrs=dict(size=80)), help_text=Messages["name"],
)
declaration_file = CodeMirrorJSONFileField(
label='Declaration',
help_text=Messages['json'],
label="Declaration", help_text=Messages["json"],
)
description_file = CodeMirrorRSTFileField(
label='Description',
label="Description",
required=False,
allow_empty_file=True,
help_text=Messages['description'],
help_text=Messages["description"],
)
class Meta:
model = ToolchainModel
exclude = []
widgets = {
'short_description': forms.TextInput(
attrs=dict(size=100),
),
'errors': forms.Textarea(
attrs=dict(readonly=1,cols=150,),
),
"short_description": forms.TextInput(attrs=dict(size=100),),
"errors": forms.Textarea(attrs=dict(readonly=1, cols=150,),),
}
def clean_declaration(self):
"""Cleans-up the file data, make sure it is really new"""
new_declaration = self.cleaned_data['declaration_file'].read()
old_declaration = ''
new_declaration = self.cleaned_data["declaration_file"].read()
old_declaration = ""
if self.instance and self.instance.declaration_file.name is not None:
old_declaration = self.instance.declaration_string
if new_declaration == old_declaration:
self.changed_data.remove('declaration_file')
self.changed_data.remove("declaration_file")
content_file = ContentFile(old_declaration)
content_file.name = self.instance.declaration_file.name
return content_file
......@@ -90,55 +80,61 @@ class ToolchainModelForm(forms.ModelForm):
# we don't validate toolchains - they should be saved in any state
# if that works out, then we return the passed file
self.cleaned_data['declaration_file'].seek(0) #reset ContentFile readout
return self.cleaned_data['declaration_file']
self.cleaned_data["declaration_file"].seek(0) # reset ContentFile readout
return self.cleaned_data["declaration_file"]
def clean(self):
"""Cleans-up the input data, make sure it overall validates"""
# make sure we don't pass back a str field as 'file'
if 'declaration_file' in self.data and \
isinstance(self.data['declaration_file'], str):
if "declaration_file" in self.data and isinstance(
self.data["declaration_file"], str
):
mutable_data = self.data.copy()
mutable_data['declaration_file'] = ContentFile(self.data['declaration_file'], name='unsaved')
mutable_data["declaration_file"] = ContentFile(
self.data["declaration_file"], name="unsaved"
)
self.data = mutable_data
#----------------------------------------------------------
# ----------------------------------------------------------
def rehash_toolchain(modeladmin, request, queryset):
"""Recalculates the hash of an toolchain"""
for q in queryset: q.save()
for q in queryset:
q.save()
rehash_toolchain.short_description = 'Rehash selected toolchains'
rehash_toolchain.short_description = "Rehash selected toolchains"
class Toolchain(admin.ModelAdmin):
list_display = ('id',
'author',
'name',
'version',
'short_description',
'creation_date',
'hash',
'previous_version',
'fork_of',
'sharing',
)
search_fields = ['author__username',
'name',
'short_description',
'previous_version__author__username',
'previous_version__name',
'fork_of__name'
]
list_display_links = ('id', 'name')
list_filter = ('sharing', )
readonly_fields = ('hash', 'errors', 'short_description')
list_display = (
"id",
"author",
"name",
"version",
"short_description",
"creation_date",
"hash",
"previous_version",
"fork_of",
"sharing",
)
search_fields = [
"author__username",
"name",
"short_description",
"previous_version__author__username",
"previous_version__name",
"fork_of__name",
]
list_display_links = ("id", "name")
list_filter = ("sharing",)
readonly_fields = ("hash", "errors", "short_description")
actions = [
rehash_toolchain,
......@@ -146,40 +142,32 @@ class Toolchain(admin.ModelAdmin):
form = ToolchainModelForm
filter_horizontal = [
'shared_with',
'shared_with_team'
]
filter_horizontal = ["shared_with", "shared_with_team"]
fieldsets = (
(None,
dict(
fields=('name', 'author'),
),
),
('Documentation',
dict(
classes=('collapse',),
fields=('short_description', 'description_file'),
),
),
('Versioning',
dict(
classes=('collapse',),
fields=('version', 'previous_version', 'fork_of'),
),
),
('Sharing',
dict(
classes=('collapse',),
fields=('sharing', 'shared_with', 'shared_with_team'),
),
),
('Source code',
dict(
fields=('hash', 'declaration_file', 'errors'),
),
),
(None, dict(fields=("name", "author"),),),
(
"Documentation",
dict(
classes=("collapse",), fields=("short_description", "description_file"),
),
),
(
"Versioning",
dict(
classes=("collapse",),
fields=("version", "previous_version", "fork_of"),
),
),
(
"Sharing",
dict(
classes=("collapse",),
fields=("sharing", "shared_with", "shared_with_team"),
),
),
("Source code", dict(fields=("hash", "declaration_file", "errors"),),),
)
admin.site.register(ToolchainModel, Toolchain)
......@@ -26,21 +26,16 @@
###############################################################################
from ..common.api import (
CheckContributionNameView,
ShareView,
ListCreateContributionView,
RetrieveUpdateDestroyContributionView,
)
from ..common.api import CheckContributionNameView
from ..common.api import ListContributionView
from ..common.api import ListCreateContributionView
from ..common.api import RetrieveUpdateDestroyContributionView
from ..common.api import ShareView
from .models import Toolchain
from .serializers import ToolchainSerializer
from .serializers import FullToolchainSerializer
from .serializers import ToolchainCreationSerializer
from .serializers import ToolchainModSerializer
from ..common.api import ListContributionView
from .serializers import ToolchainSerializer
# ----------------------------------------------------------
......
......@@ -25,18 +25,21 @@
# #
###############################################################################
from ..common.apps import CommonAppConfig
from django.utils.translation import ugettext_lazy as _
from ..common.apps import CommonAppConfig
class ToolchainsConfig(CommonAppConfig):
name = 'beat.web.toolchains'
verbose_name = _('Toolchains')
name = "beat.web.toolchains"
verbose_name = _("Toolchains")
def ready(self):
super(ToolchainsConfig, self).ready()
from .signals import auto_delete_file_on_delete, auto_delete_file_on_change
from actstream import registry
registry.register(self.get_model('Toolchain'))
from .signals import auto_delete_file_on_change # noqa: F401
from .signals import auto_delete_file_on_delete # noqa: F401
registry.register(self.get_model("Toolchain"))
......@@ -27,46 +27,163 @@
from __future__ import unicode_literals
from django.db import migrations, models
from django.conf import settings
import beat.web.toolchains.models
from django.db import migrations
from django.db import models
import beat.web.common.models
import beat.web.toolchains.models
class Migration(migrations.Migration):
dependencies = [
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('team', '0001_initial'),
("team", "0001_initial"),
]
operations = [
migrations.CreateModel(
name='Toolchain',
name="Toolchain",
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('sharing', models.CharField(default='P', max_length=1, choices=[('P', 'Private'), ('S', 'Shared'), ('A', 'Public'), ('U', 'Usable')])),
('name', models.CharField(help_text='The name for this object (space-like characters will be automatically replaced by dashes)', max_length=200)),
('version', models.PositiveIntegerField(default=1, help_text='The version of this object (an integer starting from 1)')),
('short_description', models.CharField(default='', help_text='Describe the object succinctly (try to keep it under 80 characters)', max_length=100, blank=True)),
('creation_date', models.DateTimeField(auto_now_add=True, verbose_name='Creation date')),
('hash', models.CharField(help_text='Hashed value of the object contents (<a href="https://docs.python.org/2/library/hashlib.html">SHA256, hexadecimal digest</a>). This field is auto-generated and managed by the platform.', max_length=64, editable=False)),
('declaration_file', models.FileField(db_column='declaration', upload_to=beat.web.common.models.get_contribution_declaration_filename, storage=beat.web.toolchains.models.ToolchainStorage(), max_length=200, blank=True, null=True)),
('description_file', models.FileField(db_column='description', upload_to=beat.web.common.models.get_contribution_description_filename, storage=beat.web.toolchains.models.ToolchainStorage(), max_length=200, blank=True, null=True)),
('errors', models.TextField(help_text='Errors detected while validating the toolchain. Automatically set by the platform.', null=True, blank=True)),
('author', models.ForeignKey(related_name='toolchains', to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
('fork_of', models.ForeignKey(related_name='forks', blank=True, to='toolchains.Toolchain', null=True, on_delete=models.SET_NULL)),
('previous_version', models.ForeignKey(related_name='next_versions', blank=True, to='toolchains.Toolchain', null=True, on_delete=models.SET_NULL)),
('shared_with', models.ManyToManyField(related_name='shared_toolchains', to=settings.AUTH_USER_MODEL, blank=True)),
('shared_with_team', models.ManyToManyField(related_name='shared_toolchains', to='team.Team', blank=True)),
(
"id",
models.AutoField(
verbose_name="ID",
serialize=False,
auto_created=True,
primary_key=True,
),
),
(
"sharing",
models.CharField(
default="P",
max_length=1,
choices=[
("P", "Private"),
("S", "Shared"),
("A", "Public"),
("U", "Usable"),
],
),
),
(
"name",
models.CharField(
help_text="The name for this object (space-like characters will be automatically replaced by dashes)",
max_length=200,
),
),
(
"version",
models.PositiveIntegerField(
default=1,
help_text="The version of this object (an integer starting from 1)",
),
),
(
"short_description",
models.CharField(
default="",
help_text="Describe the object succinctly (try to keep it under 80 characters)",
max_length=100,
blank=True,
),
),
(
"creation_date",
models.DateTimeField(
auto_now_add=True, verbose_name="Creation date"
),
),
(
"hash",
models.CharField(
help_text='Hashed value of the object contents (<a href="https://docs.python.org/2/library/hashlib.html">SHA256, hexadecimal digest</a>). This field is auto-generated and managed by the platform.',
max_length=64,
editable=False,
),
),
(
"declaration_file",
models.FileField(
db_column="declaration",
upload_to=beat.web.common.models.get_contribution_declaration_filename,
storage=beat.web.toolchains.models.ToolchainStorage(),
max_length=200,
blank=True,
null=True,
),
),
(
"description_file",
models.FileField(
db_column="description",
upload_to=beat.web.common.models.get_contribution_description_filename,
storage=beat.web.toolchains.models.ToolchainStorage(),
max_length=200,
blank=True,
null=True,
),
),
(
"errors",
models.TextField(
help_text="Errors detected while validating the toolchain. Automatically set by the platform.",
null=True,
blank=True,
),
),
(
"author",
models.ForeignKey(
related_name="toolchains",
to=settings.AUTH_USER_MODEL,
on_delete=models.CASCADE,
),
),
(
"fork_of",
models.ForeignKey(
related_name="forks",
blank=True,
to="toolchains.Toolchain",
null=True,
on_delete=models.SET_NULL,
),
),
(
"previous_version",
models.ForeignKey(
related_name="next_versions",
blank=True,
to="toolchains.Toolchain",
null=True,
on_delete=models.SET_NULL,
),
),
(
"shared_with",
models.ManyToManyField(
related_name="shared_toolchains",
to=settings.AUTH_USER_MODEL,
blank=True,
),
),
(
"shared_with_team",
models.ManyToManyField(
related_name="shared_toolchains", to="team.Team", blank=True
),
),
],
options={
'ordering': ['author__username', 'name', 'version'],
'abstract': False,
"ordering": ["author__username", "name", "version"],
"abstract": False,
},
),
migrations.AlterUniqueTogether(
name='toolchain',
unique_together=set([('author', 'name', 'version')]),
name="toolchain", unique_together=set([("author", "name", "version")]),
),
]
......@@ -26,7 +26,6 @@
###############################################################################
import simplejson
from django.conf import settings
from django.db import models
from django.urls import reverse
......@@ -41,8 +40,7 @@ from ..common.models import get_contribution_declaration_filename
from ..common.models import get_contribution_description_filename
from ..common.storage import OverwriteStorage
#----------------------------------------------------------
# ----------------------------------------------------------
def validate_toolchain(declaration):
......@@ -51,39 +49,49 @@ def validate_toolchain(declaration):
toolchain = beat.core.toolchain.Toolchain(settings.PREFIX, declaration)
if not toolchain.valid:
errors = 'The toolchain declaration is **invalid**. Errors:\n * ' + \
'\n * '.join(toolchain.errors)
errors = (
"The toolchain declaration is **invalid**. Errors:\n * "
+ "\n * ".join(toolchain.errors)
)
raise SyntaxError(errors)
return toolchain
#----------------------------------------------------------
# ----------------------------------------------------------
class ToolchainStorage(OverwriteStorage):
def __init__(self, *args, **kwargs):
super(ToolchainStorage, self).__init__(*args, location=settings.TOOLCHAINS_ROOT, **kwargs)
super(ToolchainStorage, self).__init__(
*args, location=settings.TOOLCHAINS_ROOT, **kwargs
)
#----------------------------------------------------------
# ----------------------------------------------------------
class ToolchainManager(StoredContributionManager):
def create_toolchain(self, author, name, short_description='', description='',
declaration=None, version=1, previous_version=None,
fork_of=None):
def create_toolchain(
self,
author,
name,
short_description="",
description="",
declaration=None,
version=1,
previous_version=None,
fork_of=None,
):
# Create the database representation of the toolchain
toolchain = self.model(
author = author,
name = self.model.sanitize_name(name),
version = version,
sharing = self.model.PRIVATE,
previous_version = previous_version,
fork_of = fork_of,
author=author,
name=self.model.sanitize_name(name),
version=version,
sharing=self.model.PRIVATE,
previous_version=previous_version,
fork_of=fork_of,
)
# Check the provided declaration
......@@ -95,11 +103,11 @@ class ToolchainManager(StoredContributionManager):
else:
tc = beat.core.toolchain.Toolchain(settings.PREFIX, data=None)
declaration = tc.data
elif not(isinstance(declaration, dict)):
elif not (isinstance(declaration, dict)):
declaration = simplejson.loads(declaration)
if len(short_description) > 0:
declaration['description'] = short_description
declaration["description"] = short_description
toolchain.declaration = declaration
......@@ -116,18 +124,18 @@ class ToolchainManager(StoredContributionManager):
toolchain.save()
if toolchain.errors:
toolchain.delete() # undo saving to respect current API
toolchain.delete() # undo saving to respect current API
return (None, toolchain.errors)
return (toolchain, None)
#----------------------------------------------------------
# ----------------------------------------------------------
class Toolchain(StoredContribution):
#_____ Constants _______
# _____ Constants _______
DEFAULT_TOOLCHAIN_TEXT = """\
{
"blocks": [],
......@@ -137,68 +145,68 @@ class Toolchain(StoredContribution):
}
"""
#_____ Fields __________
declaration_file = models.FileField(storage=ToolchainStorage(),
upload_to=get_contribution_declaration_filename,
blank=True, null=True,
max_length=200,
db_column='declaration'
)
description_file = models.FileField(storage=ToolchainStorage(),
upload_to=get_contribution_description_filename,
blank=True, null=True,
max_length=200,
db_column='description'
)
# _____ Fields __________
declaration_file = models.FileField(
storage=ToolchainStorage(),
upload_to=get_contribution_declaration_filename,
blank=True,
null=True,
max_length=200,
db_column="declaration",
)
description_file = models.FileField(
storage=ToolchainStorage(),
upload_to=get_contribution_description_filename,
blank=True,
null=True,
max_length=200,
db_column="description",
)
# read-only parameters that are updated at every save(), if required
errors = models.TextField(blank=True, null=True,
help_text="Errors detected while validating the toolchain. Automatically set by the platform.")
errors = models.TextField(
blank=True,
null=True,
help_text="Errors detected while validating the toolchain. Automatically set by the platform.",
)
objects = ToolchainManager()
#_____ Utilities __________
# _____ Utilities __________
def get_absolute_url(self):
return reverse(
'toolchains:view',
args=(self.author.username, self.name, self.version,),
"toolchains:view", args=(self.author.username, self.name, self.version,),
)
def get_api_update_url(self):
'''Returns the endpoint to update this object'''
"""Returns the endpoint to update this object"""
return reverse(
'api_toolchains:object',
args=(self.author.username, self.name, self.version,),
"api_toolchains:object",
args=(self.author.username, self.name, self.version,),
)
def get_api_share_url(self):
'''Returns the endpoint to share this object'''
"""Returns the endpoint to share this object"""
return reverse(
'api_toolchains:share',
args=(self.author.username, self.name, self.version,),
"api_toolchains:share",
args=(self.author.username, self.name, self.version,),
)
def get_new_experiment_url(self):
'''Returns the view to create a new experiment from self'''
"""Returns the view to create a new experiment from self"""
return reverse(
'experiments:new-from-toolchain',
args=(self.author.username, self.name, self.version,),
"experiments:new-from-toolchain",
args=(self.author.username, self.name, self.version,),
)
#_____ Overrides __________
# _____ Overrides __________
def save(self, *args, **kwargs):
......@@ -206,20 +214,24 @@ class Toolchain(StoredContribution):