Merge branch 'report_attestation_expiration_feature' into 'master'

[reports/attestations/utils] expiration feature

added for reports, attestations (3 reminders and cleanup). 
Simplified warning process and cleanup with a utils script. Fixes #280

See merge request !165

beat/web/attestations/management/commands/send_attestation_cleanup_warning.py

@@ -27,6 +27,7 @@
 ###############################################################################
 
 
+from django.core.management import call_command
 from django.core.management.base import BaseCommand, CommandError
 from django.core.mail import send_mail
 from django.template.loader import render_to_string
@@ -45,26 +46,29 @@ class Command(BaseCommand):
     help = 'Send email warning for attestations about to expire'
 
     def handle(self, *args, **options):
-        expiration_date = date.today() + timedelta(days=7)
-        attestations_about_to_expire = Attestation.objects.filter(locked=True, expiration_date__range=(datetime.combine(expiration_date, time.min),
-                                                                                                       datetime.combine(expiration_date, time.max)))
-        if attestations_about_to_expire:
-            current_site = Site.objects.get_current()
-            template_path = 'attestations/attestation_about_to_expire_email.txt'
-            for attestation in attestations_about_to_expire:
-                subject = "Attestation for experiment %s is about to expire" % \
-                    attestation.experiment.fullname()
+        for expiration_reminder in settings.EXPIRATION_REMINDERS:
+            expiration_date = date.today() + timedelta(days=expiration_reminder)
+            attestations_about_to_expire = Attestation.objects.filter(locked=True, expiration_date__range=(datetime.combine(expiration_date, time.min),
+                                                                                                           datetime.combine(expiration_date, time.max)))
+            if attestations_about_to_expire:
+                current_site = Site.objects.get_current()
+                template_path = 'attestations/attestation_about_to_expire_email.txt'
+                for attestation in attestations_about_to_expire:
+                    subject = "Attestation for experiment %s is about to expire" % \
+                        attestation.experiment.fullname()
 
-                send_mail(subject,
-                          render_to_string(template_path,
-                                           {
-                                             'attestation': attestation,
-                                             'beat_version': __version__,
-                                             'site': current_site,
-                                             }
-                                           ),
-                          settings.DEFAULT_FROM_EMAIL,
-                          [attestation.experiment.author.email])
-            self.stdout.write('{} attestation(s) about to expire'.format(attestations_about_to_expire.count()))
-        else:
-            self.stdout.write('No attestation(s) about to expire')
+                    send_mail(subject,
+                              render_to_string(template_path,
+                                               {
+                                                 'attestation': attestation,
+                                                 'beat_version': __version__,
+                                                 'site': current_site,
+                                                 }
+                                               ),
+                              settings.DEFAULT_FROM_EMAIL,
+                              [attestation.experiment.author.email])
+                self.stdout.write('{} attestation(s) about to expire'.format(attestations_about_to_expire.count()))
+            else:
+                self.stdout.write('No attestation(s) about to expire in {} day(s)'.format(expiration_reminder))
+
+        call_command('clean_attestations', '--noinput')

beat/web/attestations/tests.py

@@ -590,7 +590,7 @@ class CleanAttstationManagementCommandTestCase(AttestationsAPIBase):
     def test_outdated_attestation(self):
         experiment = Experiment.objects.all()[0]
         attestation = Attestation.objects.create_attestation(experiment)
-        attestation.expiration_date = attestation.expiration_date - timedelta(days=100)
+        attestation.expiration_date = attestation.expiration_date - timedelta(days=200)
         attestation.save()
         command_output = self.run_command()
         self.assertEqual(command_output, '1 attestation(s) successfully cleaned')

beat/web/reports/admin.py

@@ -84,7 +84,7 @@ class ReportAdmin(admin.ModelAdmin):
         ('Dates',
           dict(
             classes=('collapse',),
-            fields=('creation_date', 'publication_date',),
+            fields=('creation_date', 'expiration_date', 'publication_date',),
             ),
           ),
         ('Documentation',
@@ -100,7 +100,7 @@ class ReportAdmin(admin.ModelAdmin):
           ),
         )
 
-    list_display        = ('id', 'name', 'number', 'author', 'creation_date', 'publication_date')
+    list_display        = ('id', 'name', 'number', 'author', 'creation_date', 'expiration_date', 'publication_date')
     search_fields       = [
                             'author__username',
                             'name',

beat/web/reports/api.py

@@ -58,7 +58,7 @@ from ..common.exceptions import ShareError
 from ..common.mixins import CommonContextMixin
 
 from itertools import chain
-from datetime import datetime
+from datetime import datetime, timedelta
 
 
 from .permissions import IsAuthor, IsAuthorOrPublished, IsAuthorOrAccessible, IsAccessibleOutside
@@ -333,6 +333,7 @@ class LockReportView(BaseReportActionView):
             return ForbiddenResponse('Report is empty')
 
         report.status = Report.LOCKED
+        report.expiration_date = datetime.now() + timedelta(days=settings.EXPIRATION_DELTA)
         report.save()
 
         return Response(status=status.HTTP_204_NO_CONTENT)
@@ -376,6 +377,7 @@ class PublishReportView(BaseReportActionView):
 
         report.status = Report.PUBLISHED
         report.publication_date = datetime.now()
+        report.expiration_date = None
         report.save()
 
         return Response(status=status.HTTP_204_NO_CONTENT)

beat/web/reports/management/commands/clean_report.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# encoding: utf-8
+
+###############################################################################
+#                                                                             #
+# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
+# Contact: beat.support@idiap.ch                                              #
+#                                                                             #
+# This file is part of the beat.web module of the BEAT platform.              #
+#                                                                             #
+# Commercial License Usage                                                    #
+# Licensees holding valid commercial BEAT licenses may use this file in       #
+# accordance with the terms contained in a written agreement between you      #
+# and Idiap. For further information contact tto@idiap.ch                     #
+#                                                                             #
+# Alternatively, this file may be used under the terms of the GNU Affero      #
+# Public License version 3 as published by the Free Software and appearing    #
+# in the file LICENSE.AGPL included in the packaging of this file.            #
+# The BEAT platform is distributed in the hope that it will be useful, but    #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
+#                                                                             #
+# You should have received a copy of the GNU Affero Public License along      #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
+#                                                                             #
+###############################################################################
+
+
+from django.core.management.base import BaseCommand, CommandError
+
+from datetime import date
+
+from ...models import Report
+
+import sys
+import random
+
+class Command(BaseCommand):
+
+    help = 'Cleanup outdated locked reports'
+
+    def add_arguments(self, parser):
+        parser.add_argument('--noinput', action='store_false', dest='interactive', default=True,
+            help=('Tells Django to NOT prompt the user for input of any kind.'))
+
+
+    def handle(self, *args, **options):
+        clean = True
+
+        if options['interactive']:
+            try:
+                answer = self.get_input_data('Make report(s) editable again with new unique id (Y/n)? ', 'y').lower()
+            except KeyboardInterrupt:
+                self.stderr.write("\nOperation canceled.")
+                sys.exit(1)
+
+            if answer != 'y':
+                self.stdout.write('Cleanup canceled')
+                sys.exit(1)
+
+        if clean:
+            expired_reports = Report.objects.filter(status=Report.LOCKED, expiration_date__lte=date.today())
+            report_count = expired_reports.count()
+            for expired_report in expired_reports:
+                # Generate a unique report number
+                used_numbers = map(lambda x: x.number, Report.objects.all())
+                number = 0
+                while (number == 0) or number in used_numbers:
+                    number = random.randint(100000, 2**31)
+
+                expired_report.status = Report.EDITABLE
+                expired_report.number = number
+                expired_report.expiration_date = None
+                expired_report.save()
+
+            self.stdout.write('{} locked report(s) successfully cleaned'.format(report_count))
+
+
+    def get_input_data(self, message, default=None):
+        """
+        Override this method if you want to customize data inputs or
+        validation exceptions.
+        """
+        raw_value = raw_input(message)
+
+        if default and raw_value == '':
+            raw_value = default
+
+        return raw_value

beat/web/reports/management/commands/send_report_cleanup_warning_and_cleanup.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# encoding: utf-8
+
+###############################################################################
+#                                                                             #
+# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
+# Contact: beat.support@idiap.ch                                              #
+#                                                                             #
+# This file is part of the beat.web module of the BEAT platform.              #
+#                                                                             #
+# Commercial License Usage                                                    #
+# Licensees holding valid commercial BEAT licenses may use this file in       #
+# accordance with the terms contained in a written agreement between you      #
+# and Idiap. For further information contact tto@idiap.ch                     #
+#                                                                             #
+# Alternatively, this file may be used under the terms of the GNU Affero      #
+# Public License version 3 as published by the Free Software and appearing    #
+# in the file LICENSE.AGPL included in the packaging of this file.            #
+# The BEAT platform is distributed in the hope that it will be useful, but    #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
+#                                                                             #
+# You should have received a copy of the GNU Affero Public License along      #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
+#                                                                             #
+###############################################################################
+
+
+from django.core.management import call_command
+from django.core.management.base import BaseCommand, CommandError
+from django.core.mail import send_mail
+from django.template.loader import render_to_string
+from django.conf import settings
+from django.contrib.sites.models import Site
+
+from datetime import datetime, time, date, timedelta
+
+from ...models import Report
+from .... import __version__
+
+import sys
+
+class Command(BaseCommand):
+
+    help = 'Send email warning for reports about to expire and cleanup old reports'
+
+    def handle(self, *args, **options):
+        for expiration_reminder in settings.EXPIRATION_REMINDERS:
+            expiration_date = date.today() + timedelta(days=expiration_reminder)
+
+            reports_about_to_expire = Report.objects.filter(status=Report.LOCKED, expiration_date__range=(datetime.combine(expiration_date, time.min),
+                                                                                                           datetime.combine(expiration_date, time.max)))
+            if reports_about_to_expire:
+                current_site = Site.objects.get_current()
+                template_path = 'reports/report_about_to_expire_email.txt'
+                for report in reports_about_to_expire:
+                    subject = "Report %s is about to expire" % \
+                        report.name
+
+                    send_mail(subject,
+                              render_to_string(template_path,
+                                               {
+                                                 'report': report,
+                                                 'beat_version': __version__,
+                                                 'site': current_site,
+                                                 }
+                                               ),
+                              settings.DEFAULT_FROM_EMAIL,
+                              [report.author.email])
+                self.stdout.write('{} report(s) about to expire'.format(reports_about_to_expire.count()))
+            else:
+                self.stdout.write('No report(s) about to expire in {} day(s)'.format(expiration_reminder))
+
+        call_command('clean_report', '--noinput')

beat/web/reports/migrations/0002_report_expiration_date.py

+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+from datetime import datetime, timedelta
+from django.conf import settings
+
+def add_timeout_to_existing_locked_report(apps, schema_editor):
+    report_model = apps.get_model("reports", "report")
+
+    for single_report in report_model.objects.all():
+        if single_report.status == 'L':
+            single_report.expiration_date = datetime.now() + timedelta(days=settings.EXPIRATION_DELTA)
+            single_report.save()
+
+def backward_dummy(apps, schema_editor):
+    pass
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('reports', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='report',
+            name='expiration_date',
+            field=models.DateTimeField(null=True, blank=True),
+        ),
+
+        migrations.RunPython(add_timeout_to_existing_locked_report,
+                             backward_dummy)
+    ]

beat/web/reports/models.py

@@ -60,6 +60,7 @@ class ReportManager(models.Manager):
             report.content       = content
         report.creation_date     = datetime.now()
         report.publication_date  = None
+        report.expiration_date   = None
         report.status            = self.model.EDITABLE
 
         report.save()
@@ -107,6 +108,7 @@ class Report(models.Model):
     author            = models.ForeignKey(User, related_name='%(class)ss')
     experiments       = models.ManyToManyField(Experiment, related_name='reports', blank=True)
     creation_date     = models.DateTimeField()
+    expiration_date   = models.DateTimeField(null=True, blank=True)
     publication_date  = models.DateTimeField(null=True, blank=True)
     short_description = models.CharField(max_length=100, default='', blank=True, help_text=Messages['short_description'])
     description       = models.TextField(default='', blank=True)

beat/web/reports/templates/reports/report.html

@@ -134,6 +134,7 @@
 {% endif %}
 {% elif status == 'Locked' %}
 <i class="fa fa-warning"></i> This report is <strong class="text-warning">{{ status }}</strong> (not yet published)<br/>
+<i class="fa fa-calendar-o"></i> Expires in <strong>{{ report.expiration_date|naturaltime }}</strong>, on {{ report.expiration_date }} (publish it to make it permanent)<br/>
 {% endif %}
 
     </p>

beat/web/reports/templates/reports/report_about_to_expire_email.txt

+Hello,
+
+We'd like you to know that your report {{ report.name }}
+is still locked and about to expire.
+
+More details: http://{{ site.domain }}{{ report.get_author_absolute_url }}
+
+Notice that, if you don't take any action to set it to public, this 
+report will be set to editable mode on {{ report.expiration_date }}.
+
+We would like to inform you that the unique report id {{ report.get_absolute_url }}
+(accessible via this url http://{{ site.domain }}{{ report.get_absolute_url }})
+will be modified too if your report is not made public very soon.
+
+--
+https://groups.google.com/forum/#!forum/beat-devel
+BEAT version {{ beat_version }}

beat/web/settings/settings.py

@@ -170,7 +170,7 @@ ACCOUNTS_TO_EXCLUDE_FROM_TEAMS = [SYSTEM_ACCOUNT, PLOT_ACCOUNT, SCHEDULER_ACCOUN
 #
 ###########################################################################
 
-DEFAULT_FROM_EMAIL   = 'BEAT Development Platform <support@beat-eu.org>'
+DEFAULT_FROM_EMAIL   = 'BEAT Development Platform <beat.support@idiap.ch>'
 SERVER_EMAIL         = DEFAULT_FROM_EMAIL
 
 EMAIL_BACKEND        = 'django.core.mail.backends.console.EmailBackend'
@@ -351,7 +351,8 @@ REST_FRAMEWORK = {
 ##############################################################################
 
 #In days
-EXPIRATION_DELTA = 90
+EXPIRATION_DELTA = 180
+EXPIRATION_REMINDERS = [1, 7, 30]
 
 ##############################################################################
 #

beat/web/utils/management/commands/daily_cron_actions.py

+#!/usr/bin/env python
+# vim: set fileencoding=utf-8 :
+# encoding: utf-8
+
+###############################################################################
+#                                                                             #
+# Copyright (c) 2016 Idiap Research Institute, http://www.idiap.ch/           #
+# Contact: beat.support@idiap.ch                                              #
+#                                                                             #
+# This file is part of the beat.web module of the BEAT platform.              #
+#                                                                             #
+# Commercial License Usage                                                    #
+# Licensees holding valid commercial BEAT licenses may use this file in       #
+# accordance with the terms contained in a written agreement between you      #
+# and Idiap. For further information contact tto@idiap.ch                     #
+#                                                                             #
+# Alternatively, this file may be used under the terms of the GNU Affero      #
+# Public License version 3 as published by the Free Software and appearing    #
+# in the file LICENSE.AGPL included in the packaging of this file.            #
+# The BEAT platform is distributed in the hope that it will be useful, but    #
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY  #
+# or FITNESS FOR A PARTICULAR PURPOSE.                                        #
+#                                                                             #
+# You should have received a copy of the GNU Affero Public License along      #
+# with the BEAT platform. If not, see http://www.gnu.org/licenses/.           #
+#                                                                             #
+###############################################################################
+
+
+from django.core.management import call_command
+from django.core.management.base import BaseCommand, CommandError
+from django.core.mail import send_mail
+from django.template.loader import render_to_string
+from django.conf import settings
+from django.contrib.sites.models import Site
+
+from datetime import datetime, time, date, timedelta
+
+from ....reports.models import Report
+from .... import __version__
+
+import sys
+
+class Command(BaseCommand):
+
+    help = 'Daily CRON actions'
+
+    def handle(self, *args, **options):
+        
+        # Send attestations cleanup warnings and cleanup attestations
+        call_command('send_attestation_cleanup_warning')
+
+        # Send report cleanup warnings and cleanup reports
+        call_command('send_report_cleanup_warning_and_cleanup')