Commit 8d888733 authored by Flavio TARSETTI's avatar Flavio TARSETTI
Browse files

Merge branch 'cleanup_search' into 'django3_migration'

Cleanup search

See merge request !361
parents 6efb94ac f02be244
Pipeline #42670 passed with stage
in 15 minutes and 33 seconds
......@@ -25,4 +25,4 @@
# #
###############################################################################
default_app_config = 'beat.web.search.apps.SearchConfig'
default_app_config = "beat.web.search.apps.SearchConfig"
......@@ -25,18 +25,18 @@
# #
###############################################################################
from django.contrib import admin
from django import forms
from django.contrib import admin
from ..common.texts import Messages
from .models import Search, Leaderboard, Rank
from ..ui.forms import CodeMirrorRSTCharField
from ..ui.forms import CodeMirrorJSONCharField
from ..ui.forms import CodeMirrorRSTCharField
from ..ui.forms import NameField
from .models import Leaderboard
from .models import Rank
from .models import Search
#----------------------------------------------------------
# ----------------------------------------------------------
def rehash_search(modeladmin, request, queryset):
......@@ -44,64 +44,57 @@ def rehash_search(modeladmin, request, queryset):
for q in queryset:
q.save()
rehash_search.short_description = 'Rehash selected search'
rehash_search.short_description = "Rehash selected search"
#----------------------------------------------------------
# ----------------------------------------------------------
class SearchModelForm(forms.ModelForm):
name = NameField(
widget=forms.TextInput(attrs=dict(size=80)),
help_text=Messages['algo_name'],
widget=forms.TextInput(attrs=dict(size=80)), help_text=Messages["algo_name"],
)
description = CodeMirrorRSTCharField(
required=False,
help_text=Messages['description'],
required=False, help_text=Messages["description"],
)
filters = CodeMirrorJSONCharField(
readonly=False,
required=False,
help_text=Messages['json'],
readonly=False, required=False, help_text=Messages["json"],
)
settings = CodeMirrorJSONCharField(
readonly=False,
required=False,
help_text=Messages['json'],
readonly=False, required=False, help_text=Messages["json"],
)
class Meta:
model = Search
exclude = []
widgets = {
'short_description': forms.TextInput(
attrs=dict(size=100),
),
"short_description": forms.TextInput(attrs=dict(size=100),),
}
#----------------------------------------------------------
# ----------------------------------------------------------
class SearchAdmin(admin.ModelAdmin):
list_display = ('id', 'author', 'name')
search_fields = [
'author__username',
'name',
'short_description',
'description',
'filters',
'settings',
list_display = ("id", "author", "name")
search_fields = [
"author__username",
"name",
"short_description",
"description",
"filters",
"settings",
]
list_display_links = ('id', 'name')
list_display_links = ("id", "name")
list_filter = ('author', 'name', 'version')
readonly_fields = ('hash',)
list_filter = ("author", "name", "version")
readonly_fields = ("hash",)
actions = [
rehash_search,
......@@ -109,87 +102,98 @@ class SearchAdmin(admin.ModelAdmin):
form = SearchModelForm
filter_horizontal = [
'shared_with',
'shared_with_team'
]
filter_horizontal = ["shared_with", "shared_with_team"]
fieldsets = (
(None,
dict(
fields=('name', 'author',),
),
),
('Documentation',
dict(
classes=('collapse',),
fields=('short_description', 'description',),
),
),
('Versioning',
dict(
classes=('collapse',),
fields=('version', 'previous_version', 'fork_of'),
),
),
('Sharing',
dict(
classes=('collapse',),
fields=('sharing', 'shared_with', 'shared_with_team'),
),
),
('Definition',
dict(
fields=('hash', 'filters', 'settings'),
),
),
(None, dict(fields=("name", "author",),),),
(
"Documentation",
dict(classes=("collapse",), fields=("short_description", "description",),),
),
(
"Versioning",
dict(
classes=("collapse",),
fields=("version", "previous_version", "fork_of"),
),
),
(
"Sharing",
dict(
classes=("collapse",),
fields=("sharing", "shared_with", "shared_with_team"),
),
),
("Definition", dict(fields=("hash", "filters", "settings"),),),
)
admin.site.register(Search, SearchAdmin)
class RankInline(admin.TabularInline):
model = Rank
can_delete = False
extra = 0
max_num = 0
readonly_fields = ('id', 'algorithm', 'result', 'order', 'experiment')
ordering = ('algorithm', 'order',)
model = Rank
can_delete = False
extra = 0
max_num = 0
readonly_fields = ("id", "algorithm", "result", "order", "experiment")
ordering = (
"algorithm",
"order",
)
def number_of_experiments(obj):
return obj.experiments.count()
number_of_experiments.short_description = 'Experiments'
number_of_experiments.short_description = "Experiments"
def users_to_notify(obj):
return obj.notify.count()
users_to_notify.short_description = 'Subscribed'
users_to_notify.short_description = "Subscribed"
def search_sharing(obj):
return obj.search.get_sharing_display()
search_sharing.short_description = 'Sharing'
search_sharing.short_description = "Sharing"
class LeaderboardAdmin(admin.ModelAdmin):
list_display = ('id', 'search', 'created', 'updated', users_to_notify, number_of_experiments, search_sharing)
list_display = (
"id",
"search",
"created",
"updated",
users_to_notify,
number_of_experiments,
search_sharing,
)
search_fiels = [
'search__author__username',
'search__name',
'search_short_description',
'search_description',
'filters',
'settings',
"search__author__username",
"search__name",
"search_short_description",
"search_description",
"filters",
"settings",
]
list_display_links = ('id', 'search',)
list_display_links = (
"id",
"search",
)
inlines = [
RankInline,
]
filter_horizontal = [
'notify',
"notify",
]
admin.site.register(Leaderboard, LeaderboardAdmin)
......@@ -26,46 +26,40 @@
###############################################################################
import simplejson as json
from functools import reduce
import simplejson as json
from django.conf import settings
from django.contrib.auth.models import User
from django.db.models import Q
from django.shortcuts import get_object_or_404
from rest_framework.response import Response
from rest_framework.views import APIView
from rest_framework import permissions as drf_permissions
from rest_framework import generics
from rest_framework import permissions as drf_permissions
from rest_framework import status
from rest_framework.response import Response
from rest_framework.views import APIView
from ..algorithms.models import Algorithm
from ..common import permissions as beat_permissions
from ..common.api import ShareView
from ..common.mixins import CommonContextMixin
from ..common.mixins import SerializerFieldsMixin
from ..common.models import Shareable
from ..common.responses import BadRequestResponse
from ..common.utils import ensure_html
from ..common.utils import py3_cmp
from ..databases.models import Database
from ..dataformats.models import DataFormat
from ..experiments.models import Experiment
from ..toolchains.models import Toolchain
from ..common.models import Shareable
from ..common.api import ShareView
from ..common.utils import ensure_html
from ..common.responses import BadRequestResponse
from ..common.mixins import CommonContextMixin, SerializerFieldsMixin
from ..common.utils import py3_cmp
from ..common import permissions as beat_permissions
from ..ui.templatetags.gravatar import gravatar_hash
from .utils import apply_filter
from .utils import FilterGenerator
from .utils import OR
from .models import Search
from .serializers import SearchResultSerializer, SearchSerializer, SearchWriteSerializer
from .serializers import SearchResultSerializer
from .serializers import SearchSerializer
from .serializers import SearchWriteSerializer
from .utils import OR
from .utils import FilterGenerator
from .utils import apply_filter
# ------------------------------------------------
......@@ -111,9 +105,9 @@ class SearchView(APIView):
filters = None
display_settings = None
if 'query' in data:
if not(isinstance(data['query'], str)) or (len(data['query']) == 0):
return BadRequestResponse('Invalid query data')
if "query" in data:
if not (isinstance(data["query"], str)) or (len(data["query"]) == 0):
return BadRequestResponse("Invalid query data")
query = data["query"]
else:
......
......@@ -25,14 +25,17 @@
# #
###############################################################################
from ..common.apps import CommonAppConfig
from django.utils.translation import ugettext_lazy as _
from ..common.apps import CommonAppConfig
class SearchConfig(CommonAppConfig):
name = 'beat.web.search'
verbose_name = _('Search')
name = "beat.web.search"
verbose_name = _("Search")
def ready(self):
super(SearchConfig, self).ready()
from actstream import registry
registry.register(self.get_model('Search'))
registry.register(self.get_model("Search"))
......@@ -27,5 +27,6 @@
from rest_framework import serializers
class DictListField(serializers.ListField):
child = serializers.DictField()
......@@ -28,32 +28,33 @@
import logging
logger = logging.getLogger(__name__)
from django.core.management.base import BaseCommand
from django.core.mail import EmailMessage
from django.template.loader import render_to_string
from django.conf import settings
from django.contrib.sites.models import Site
from django.core.mail import EmailMessage
from django.core.management.base import BaseCommand
from django.template.loader import render_to_string
from ...models import Leaderboard
from .... import __version__
from ...models import Leaderboard
logger = logging.getLogger(__name__)
import sys
import datetime
class Command(BaseCommand):
help = 'Update and (optionally) notify owners on leaderboard changes'
help = "Update and (optionally) notify owners on leaderboard changes"
def handle(self, *ignored, **arguments):
# Setup this command's logging level
global logger
arguments['verbosity'] = int(arguments['verbosity'])
if arguments['verbosity'] >= 1:
if arguments['verbosity'] == 1: logger.setLevel(logging.INFO)
elif arguments['verbosity'] >= 2: logger.setLevel(logging.DEBUG)
arguments["verbosity"] = int(arguments["verbosity"])
if arguments["verbosity"] >= 1:
if arguments["verbosity"] == 1:
logger.setLevel(logging.INFO)
elif arguments["verbosity"] >= 2:
logger.setLevel(logging.DEBUG)
objects_updated = 0
objects_changed = 0
......@@ -61,28 +62,32 @@ class Command(BaseCommand):
for obj in Leaderboard.objects.all():
prev_date = obj.updated
logger.info('Updating leaderboard for search %s...' % obj.search)
logger.info("Updating leaderboard for search %s..." % obj.search)
changed = obj.update_experiments()
objects_updated += 1
emails = obj.notify.values_list('email', flat=True)
emails = obj.notify.values_list("email", flat=True)
# compare tables and notify if required
if changed and emails:
logger.info('Notifying interested parties for %s' % obj.search)
logger.info("Notifying interested parties for %s" % obj.search)
objects_changed += 1
current_site = Site.objects.get_current()
template_path = 'search/leaderboard_changed.txt'
subject = "Experiment ranking for leaderboard \"%s\" changed" % obj.search.fullname()
template_path = "search/leaderboard_changed.txt"
subject = (
'Experiment ranking for leaderboard "%s" changed'
% obj.search.fullname()
)
mesg = EmailMessage(
subject.strip(),
render_to_string(template_path,
{
'leaderboard': obj,
'prev_date': prev_date,
'beat_version': __version__,
'site': current_site,
}
render_to_string(
template_path,
{
"leaderboard": obj,
"prev_date": prev_date,
"beat_version": __version__,
"site": current_site,
},
),
settings.DEFAULT_FROM_EMAIL,
to=[],
......@@ -91,7 +96,11 @@ class Command(BaseCommand):
mesg.send()
else:
logger.debug('Leaderboard %s did not change or notification is off' % obj.search)
logger.debug(
"Leaderboard %s did not change or notification is off" % obj.search
)
logger.info('Leaderboard updates: %d / Notifications: %d' % \
(objects_updated, objects_changed))
logger.info(
"Leaderboard updates: %d / Notifications: %d"
% (objects_updated, objects_changed)
)
......@@ -27,77 +27,216 @@
from __future__ import unicode_literals
from django.db import migrations, models
import datetime
from django.conf import settings
from django.db import migrations
from django.db import models
class Migration(migrations.Migration):
dependencies = [
('experiments', '0001_initial'),
('algorithms', '0001_initial'),
("experiments", "0001_initial"),
("algorithms", "0001_initial"),
migrations.swappable_dependency(settings.AUTH_USER_MODEL),
('team', '0001_initial'),
("team", "0001_initial"),
]
operations = [
migrations.CreateModel(
name='Leaderboard',
name="Leaderboard",
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('created', models.DateTimeField(auto_now_add=True)),
('updated', models.DateTimeField(auto_now=True)),
('changed', models.DateTimeField(default=datetime.datetime.now)),
(
"id",
models.AutoField(
verbose_name="ID",
serialize=False,
auto_created=True,
primary_key=True,
),
),
("created", models.DateTimeField(auto_now_add=True)),
("updated", models.DateTimeField(auto_now=True)),
("changed", models.DateTimeField(default=datetime.datetime.now)),
],
),
migrations.CreateModel(
name='Rank',
name="Rank",
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('order', models.PositiveIntegerField()),
('algorithm', models.ForeignKey(to='algorithms.Algorithm', on_delete=models.CASCADE)),
('experiment', models.ForeignKey(to='experiments.Experiment', on_delete=models.CASCADE)),
('leaderboard', models.ForeignKey(to='search.Leaderboard', on_delete=models.CASCADE)),
('result', models.ManyToManyField(to='experiments.Result')),
(
"id",
models.AutoField(
verbose_name="ID",
serialize=False,
auto_created=True,
primary_key=True,
),
),
("order", models.PositiveIntegerField()),
(
"algorithm",
models.ForeignKey(
to="algorithms.Algorithm", on_delete=models.CASCADE
),
),
(
"experiment",
models.ForeignKey(
to="experiments.Experiment", on_delete=models.CASCADE
),
),
(
"leaderboard",
models.ForeignKey(
to="search.Leaderboard", on_delete=models.CASCADE
),
),
("result", models.ManyToManyField(to="experiments.Result")),
],
),
migrations.CreateModel(
name='Search',
name="Search",
fields=[
('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
('sharing', models.CharField(default=b'P', max_length=1, choices=[(b'P', b'Private'), (b'S', b'Shared'), (b'A', b'Public'), (b'U', b'Usable')])),
('name', models.CharField(help_text=b'The name for this object (space-like characters will be automatically replaced by dashes)', max_length=200)),
('version', models.PositiveIntegerField(default=1, help_text=b'The version of this object (an integer starting from 1)')),
('short_description', models.CharField(default=b'', help_text=b'Describe the object succinctly (try to keep it under 80 characters)', max_length=100, blank=True)),
('creation_date', models.DateTimeField(auto_now_add=True, verbose_name=b'Creation date')),
('hash', models.CharField(help_text=b'Hashed value of the object contents (<a href="https://docs.python.org/2/library/hashlib.html">SHA256, hexadecimal digest</a>). This field is auto-generated and managed by the platform.', max_length=64, editable=False)),
('filters', models.TextField(default=b'', blank=True)),
('settings', models.TextField(default=b'', blank=True)),
('description', models.TextField(default=b'', help_text=b'Describe the object thoroughly using <a href="http://docutils.sourceforge.net/rst.html">reStructuredText mark-up</a><br/><i class="fa fa-thumbs-up"></i> The ruler at 80 columns indicate suggested <a href="https://en.wikipedia.org/wiki/POSIX">POSIX line breaks</a> (for readability).<br/><i class="fa fa-thumbs-up"></i> The editor will automatically enlarge to accomodate the entirety of your input<br/><i class="fa fa-thumbs-up"></i> Use <a href="http://codemirror.net/doc/manual.html#commands">keyboard shortcuts</a> for search/replace and faster editing. For example, use Ctrl-F (PC) or Cmd-F (Mac) to search through this box', blank=True)),
('author', models.ForeignKey(related_name='searchs', to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)),
('fork_of', models.ForeignKey(related_name='forks', blank=True, to='search.Search', null=True, on_delete=models.SET_NULL)),
('previous_version', models.ForeignKey(related_name='next_versions', blank=True, to='search.Search', null=True, on_delete=models.SET_NULL)),
('shared_with', models.ManyToManyField(related_name='shared_searchs', to=settings.AUTH_USER_MODEL, blank=True)),
('shared_with_team', models.ManyToManyField(related_name='shared_searchs', to='team.Team', blank=True)),
(
"id",
models.AutoField(
verbose_name="ID",
serialize=False,
auto_created=True,
primary_key=True,
),
),
(
"sharing",
models.CharField(
default=b"P",
max_length=1,
choices=[
(b"P", b"Private"),