Skip to content
GitLab
Commit 8fa617f2 authored by Flavio TARSETTI's avatar Flavio TARSETTI
Browse files

Merge branch 'cleanup_reports' into 'django3_migration' 

Cleanup reports

See merge request !357
parents ea81206d 95a2b05d
Pipeline #42754 passed with stage
in 15 minutes and 9 seconds
Show whitespace changes
Inline Side-by-side
beat/web/reports/admin.py
View file @ 8fa617f2
#!/usr/bin/env python
# vim: set fileencoding=utf-8 :
#!/usr/bin/env python
# encoding: utf-8
###############################################################################
......@@ -30,86 +28,92 @@
# Flavio Tarsetti <flavio.tarsetti@idiap.ch>
# Wed May 20 15:46:11 CEST 2015
from django.contrib import admin
import simplejson as json
from django import forms
from django.contrib import admin
from ..common.texts import Messages
from ..ui.forms import CodeMirrorJSONCharField
from ..ui.forms import CodeMirrorRSTCharField
from .models import Report
from ..ui.forms import CodeMirrorRSTCharField, CodeMirrorJSONCharField
from ..ui.forms import NameField
import simplejson as json
class ReportModelForm(forms.ModelForm):
description = CodeMirrorRSTCharField(
label='Description',
required=False,
help_text=Messages['description'],
label="Description", required=False, help_text=Messages["description"],
)
content = CodeMirrorJSONCharField(
label='Content',
help_text=Messages['json'],
)
content = CodeMirrorJSONCharField(label="Content", help_text=Messages["json"],)
def clean_content(self):
"""Cleans-up the content data, make sure it is really new"""
try:
data = json.loads(self.cleaned_data['content'])
data = json.loads(self.cleaned_data["content"])
return json.dumps(data, indent=4)
except Exception as e:
raise forms.ValidationError(str(e))
class ReportAdmin(admin.ModelAdmin):
form = ReportModelForm
readonly_fields = ('referenced_plotters', 'referenced_plotterparameters')
readonly_fields = ("referenced_plotters", "referenced_plotterparameters")
filter_horizontal = [
'experiments',
'referenced_plotters',
'referenced_plotterparameters'
"experiments",
"referenced_plotters",
"referenced_plotterparameters",
]
fieldsets = (
(None,
(None, dict(fields=("status", "name", "number", "author",),),),
(
"Dates",
dict(
fields=('status', 'name', 'number', 'author',),
classes=("collapse",),
fields=("creation_date", "expiration_date", "publication_date",),
),
),
('Dates',
dict(
classes=('collapse',),
fields=('creation_date', 'expiration_date', 'publication_date',),
),
(
"Documentation",
dict(classes=("collapse",), fields=("short_description", "description",),),
),
('Documentation',
(
None,
dict(
classes=('collapse',),
fields=('short_description', 'description',),
),
fields=(
"analyzer",
"experiments",
"referenced_plotters",
"referenced_plotterparameters",
"content",
),
(None,
dict(
fields=('analyzer', 'experiments', 'referenced_plotters', 'referenced_plotterparameters', 'content',),
),
),
)
list_display = ('id', 'name', 'number', 'author', 'creation_date', 'expiration_date', 'publication_date')
list_display = (
"id",
"name",
"number",
"author",
"creation_date",
"expiration_date",
"publication_date",
)
search_fields = [
'author__username',
'name',
'short_description',
'description',
'number',
"author__username",
"name",
"short_description",
"description",
"number",
]
list_display_links = ('id', 'name')
list_display_links = ("id", "name")
list_filter = ("author", "name")
list_filter = ('author', 'name')
admin.site.register(Report, ReportAdmin)
beat/web/reports/api.py
View file @ 8fa617f2
......@@ -26,44 +26,38 @@
###############################################################################
import json
from datetime import datetime, timedelta
from datetime import datetime
from datetime import timedelta
from django.conf import settings
from django.contrib.auth.models import User
from django.shortcuts import get_object_or_404
from django.db.models import Q
from django.shortcuts import get_object_or_404
from django.urls import reverse
from rest_framework import generics
from rest_framework import views
from rest_framework import permissions
from rest_framework.response import Response
from rest_framework import status
from rest_framework import views
from rest_framework.response import Response
from ..common.models import Shareable
from ..common.exceptions import ShareError
from ..common.mixins import CommonContextMixin
from ..common.responses import BadRequestResponse, ForbiddenResponse
from ..common.models import Shareable
from ..common.responses import BadRequestResponse
from ..common.responses import ForbiddenResponse
from ..common.utils import ensure_html
from ..experiments.serializers import ExperimentResultsSerializer
from .serializers import SimpleReportSerializer
from .models import Report
from .permissions import IsAccessibleOutside
from .permissions import IsAuthor
from .permissions import IsAuthorOrPublished
from .permissions import IsEditable
from .permissions import IsLocked
from .serializers import FullReportSerializer
from .serializers import ReportCreationFailedException
from .serializers import ReportCreationSerializer
from .serializers import ReportUpdateSerializer
from .serializers import ReportCreationFailedException
from .models import Report
from .permissions import (
IsAuthor,
IsAuthorOrPublished,
IsAccessibleOutside,
IsEditable,
IsLocked,
)
from .serializers import SimpleReportSerializer
# ----------------------------------------------------------
......
beat/web/reports/management/commands/send_report_cleanup_warning_and_cleanup.py
View file @ 8fa617f2
......@@ -27,49 +27,66 @@
###############################################################################
from django.core.management import call_command
from django.core.management.base import BaseCommand, CommandError
from django.core.mail import send_mail
from django.template.loader import render_to_string
from datetime import date
from datetime import datetime
from datetime import time
from datetime import timedelta
from django.conf import settings
from django.contrib.sites.models import Site
from django.core.mail import send_mail
from django.core.management import call_command
from django.core.management.base import BaseCommand
from django.template.loader import render_to_string
from datetime import datetime, time, date, timedelta
from ...models import Report
from .... import __version__
from ...models import Report
import sys
class Command(BaseCommand):
help = 'Send email warning for reports about to expire and cleanup old reports'
help = "Send email warning for reports about to expire and cleanup old reports"
def handle(self, *args, **options):
for expiration_reminder in settings.EXPIRATION_REMINDERS:
expiration_date = date.today() + timedelta(days=expiration_reminder)
reports_about_to_expire = Report.objects.filter(status=Report.LOCKED, expiration_date__range=(datetime.combine(expiration_date, time.min),
datetime.combine(expiration_date, time.max)))
reports_about_to_expire = Report.objects.filter(
status=Report.LOCKED,
expiration_date__range=(
datetime.combine(expiration_date, time.min),
datetime.combine(expiration_date, time.max),
),
)
if reports_about_to_expire:
current_site = Site.objects.get_current()
template_path = 'reports/report_about_to_expire_email.txt'
template_path = "reports/report_about_to_expire_email.txt"
for report in reports_about_to_expire:
subject = "Report %s is about to expire" % \
report.name
subject = "Report %s is about to expire" % report.name
send_mail(subject,
render_to_string(template_path,
send_mail(
subject,
render_to_string(
template_path,
{
'report': report,
'beat_version': __version__,
'site': current_site,
}
"report": report,
"beat_version": __version__,
"site": current_site,
},
),
settings.DEFAULT_FROM_EMAIL,
[report.author.email])
self.stdout.write('{} report(s) about to expire'.format(reports_about_to_expire.count()))
[report.author.email],
)
self.stdout.write(
"{} report(s) about to expire".format(
reports_about_to_expire.count()
)
)
else:
self.stdout.write('No report(s) about to expire in {} day(s)'.format(expiration_reminder))
self.stdout.write(
"No report(s) about to expire in {} day(s)".format(
expiration_reminder
)
)
call_command('clean_report', '--noinput')
call_command("clean_report", "--noinput")
beat/web/reports/migrations/0001_initial.py
View file @ 8fa617f2
......@@ -27,37 +27,104 @@
from __future__ import unicode_literals
from django.db import migrations, models
from django.conf import settings
from django.db import migrations
from django.db import models
class Migration(migrations.Migration):
dependencies = [
('experiments', '0001_initial'),
('algorithms', '0001_initial'),
("experiments", "0001_initial"),
("algorithms", "0001_initial"),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('plotters', '0001_initial'),
("plotters", "0001_initial"),
]
operations = [
migrations.CreateModel(
name='Report',
name="Report",
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('status', models.CharField(default=b'E', max_length=1, choices=[(b'E', b'Editable'), (b'L', b'Locked'), (b'P', b'Published')])),
('name', models.CharField(help_text=b'The name for this object (space-like characters will be automatically replaced by dashes)', max_length=200)),
('number', models.IntegerField(blank=True)),
('creation_date', models.DateTimeField()),
('publication_date', models.DateTimeField(null=True, blank=True)),
('short_description', models.CharField(default=b'', help_text=b'Describe the object succinctly (try to keep it under 80 characters)', max_length=100, blank=True)),
('description', models.TextField(default=b'', blank=True)),
('content', models.TextField(default=b'{}', blank=True)),
('analyzer', models.ForeignKey(related_name='reports', blank=True, to='algorithms.Algorithm', null=True, on_delete=models.SET_NULL)),
('author', models.ForeignKey(related_name='reports', to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
('experiments', models.ManyToManyField(related_name='reports', to='experiments.Experiment', blank=True)),
('referenced_plotterparameters', models.ManyToManyField(related_name='reports', to='plotters.PlotterParameter', blank=True)),
('referenced_plotters', models.ManyToManyField(related_name='reports', to='plotters.Plotter', blank=True)),
(
"id",
models.AutoField(
verbose_name="ID",
serialize=False,
auto_created=True,
primary_key=True,
),
),
(
"status",
models.CharField(
default=b"E",
max_length=1,
choices=[
(b"E", b"Editable"),
(b"L", b"Locked"),
(b"P", b"Published"),
],
),
),
(
"name",
models.CharField(
help_text=b"The name for this object (space-like characters will be automatically replaced by dashes)",
max_length=200,
),
),
("number", models.IntegerField(blank=True)),
("creation_date", models.DateTimeField()),
("publication_date", models.DateTimeField(null=True, blank=True)),
(
"short_description",
models.CharField(
default=b"",
help_text=b"Describe the object succinctly (try to keep it under 80 characters)",
max_length=100,
blank=True,
),
),
("description", models.TextField(default=b"", blank=True)),
("content", models.TextField(default=b"{}", blank=True)),
(
"analyzer",
models.ForeignKey(
related_name="reports",
blank=True,
to="algorithms.Algorithm",
null=True,
on_delete=models.SET_NULL,
),
),
(
"author",
models.ForeignKey(
related_name="reports",
to=settings.AUTH_USER_MODEL,
on_delete=models.CASCADE,
),
),
(
"experiments",
models.ManyToManyField(
related_name="reports", to="experiments.Experiment", blank=True
),
),
(
"referenced_plotterparameters",
models.ManyToManyField(
related_name="reports",
to="plotters.PlotterParameter",
blank=True,
),
),
(
"referenced_plotters",
models.ManyToManyField(
related_name="reports", to="plotters.Plotter", blank=True
),
),
],
),
]
beat/web/reports/migrations/0002_report_expiration_date.py
View file @ 8fa617f2
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
from datetime import datetime
from datetime import timedelta
from datetime import datetime, timedelta
from django.conf import settings
from django.db import migrations
from django.db import models
def add_timeout_to_existing_locked_report(apps, schema_editor):
report_model = apps.get_model("reports", "report")
for single_report in report_model.objects.all():
if single_report.status == 'L':
single_report.expiration_date = datetime.now() + timedelta(days=settings.EXPIRATION_DELTA)
if single_report.status == "L":
single_report.expiration_date = datetime.now() + timedelta(
days=settings.EXPIRATION_DELTA
)
single_report.save()
def backward_dummy(apps, schema_editor):
pass
class Migration(migrations.Migration):
dependencies = [
('reports', '0001_initial'),
("reports", "0001_initial"),
]
operations = [
migrations.AddField(
model_name='report',
name='expiration_date',
model_name="report",
name="expiration_date",
field=models.DateTimeField(null=True, blank=True),
),
migrations.RunPython(add_timeout_to_existing_locked_report,
backward_dummy)
migrations.RunPython(add_timeout_to_existing_locked_report, backward_dummy),
]
beat/web/reports/migrations/0003_report_last_edited_date.py
View file @ 8fa617f2
......@@ -2,19 +2,20 @@
# Generated by Django 1.9.5 on 2017-03-13 15:11
from __future__ import unicode_literals
from django.db import migrations, models
from django.db import migrations
from django.db import models
class Migration(migrations.Migration):
dependencies = [
('reports', '0002_report_expiration_date'),
("reports", "0002_report_expiration_date"),
]
operations = [
migrations.AddField(
model_name='report',
name='last_edited_date',
model_name="report",
name="last_edited_date",
field=models.DateTimeField(null=True),
),
]
beat/web/reports/migrations/0004_auto_20170410_1121.py
View file @ 8fa617f2
......@@ -2,65 +2,77 @@
# Generated by Django 1.9.5 on 2017-04-10 11:21
from __future__ import unicode_literals
from django.db import migrations
import json
import re
from django.db import migrations
# parses a table from the old representation (a list of column objects)
# and returns a table in the new representation
def parse_table(table, precision):
conv_table = {
'itemName': 'Table',
'fields': ['Experiment'],
'precision': precision
}
conv_table = {"itemName": "Table", "fields": ["Experiment"], "precision": precision}
for row in table:
if not row['selected']:
if not row["selected"]:
continue
name = row['name']
name = re.sub(r'\[.*\]$', '', name)
name = row["name"]
name = re.sub(r"\[.*\]$", "", name)
if name == 'experiment':
if name == "experiment":
continue
if name == 'experiment.execution_time' or name == 'execution_time':
name = 'total_execution_time'
elif re.match(r'^execution_time\.', name):
name = re.sub(r'execution_time', 'linear_execution_time', name)
if name == "experiment.execution_time" or name == "execution_time":
name = "total_execution_time"
elif re.match(r"^execution_time\.", name):
name = re.sub(r"execution_time", "linear_execution_time", name)
if name.startswith('experiment.'):
name = re.sub(r'experiment\.', '', name)
if name.startswith("experiment."):
name = re.sub(r"experiment\.", "", name)
if '.' in name:
segs = name.split('.')
name = segs[1] + '.' + segs[0]
conv_table['fields'].append(name)
if "." in name:
segs = name.split(".")
name = segs[1] + "." + segs[0]
conv_table["fields"].append(name)
return conv_table
# parses a plot from the old representation
# and returns a plot in the new representation
def parse_plot(plot):
conv_plot = {
'itemName': plot['required_plotter'][0],
'name': plot['data']['output'][0],
'type': plot['required_plotter'][0]
"itemName": plot["required_plotter"][0],
"name": plot["data"]["output"][0],
"type": plot["required_plotter"][0],
}
return conv_plot
# helper func to build the experiment's full name
def experiment_fullname(exp):
return '%s/%s/%s/%s/%s' % (exp.author.username, exp.toolchain.author.username, exp.toolchain.name, exp.toolchain.version, exp.name)
return "%s/%s/%s/%s/%s" % (
exp.author.username,
exp.toolchain.author.username,
exp.toolchain.name,
exp.toolchain.version,
exp.name,
)
# helper func to build the analyzer's full name
def analyzer_fullname(report):
return '%s/%s/%s' % (report.analyzer.author.username, report.analyzer.name, report.analyzer.version)
return "%s/%s/%s" % (
report.analyzer.author.username,
report.analyzer.name,
report.analyzer.version,
)