Commit 8d888733 authored by Flavio TARSETTI's avatar Flavio TARSETTI

Merge branch 'cleanup_search' into 'django3_migration'

Cleanup search

See merge request !361
parents 6efb94ac f02be244
Pipeline #42670 passed with stage
in 15 minutes and 33 seconds
......@@ -25,4 +25,4 @@
# #
###############################################################################
default_app_config = 'beat.web.search.apps.SearchConfig'
default_app_config = "beat.web.search.apps.SearchConfig"
......@@ -25,18 +25,18 @@
# #
###############################################################################
from django.contrib import admin
from django import forms
from django.contrib import admin
from ..common.texts import Messages
from .models import Search, Leaderboard, Rank
from ..ui.forms import CodeMirrorRSTCharField
from ..ui.forms import CodeMirrorJSONCharField
from ..ui.forms import CodeMirrorRSTCharField
from ..ui.forms import NameField
from .models import Leaderboard
from .models import Rank
from .models import Search
#----------------------------------------------------------
# ----------------------------------------------------------
def rehash_search(modeladmin, request, queryset):
......@@ -44,64 +44,57 @@ def rehash_search(modeladmin, request, queryset):
for q in queryset:
q.save()
rehash_search.short_description = 'Rehash selected search'
rehash_search.short_description = "Rehash selected search"
#----------------------------------------------------------
# ----------------------------------------------------------
class SearchModelForm(forms.ModelForm):
name = NameField(
widget=forms.TextInput(attrs=dict(size=80)),
help_text=Messages['algo_name'],
widget=forms.TextInput(attrs=dict(size=80)), help_text=Messages["algo_name"],
)
description = CodeMirrorRSTCharField(
required=False,
help_text=Messages['description'],
required=False, help_text=Messages["description"],
)
filters = CodeMirrorJSONCharField(
readonly=False,
required=False,
help_text=Messages['json'],
readonly=False, required=False, help_text=Messages["json"],
)
settings = CodeMirrorJSONCharField(
readonly=False,
required=False,
help_text=Messages['json'],
readonly=False, required=False, help_text=Messages["json"],
)
class Meta:
model = Search
exclude = []
widgets = {
'short_description': forms.TextInput(
attrs=dict(size=100),
),
"short_description": forms.TextInput(attrs=dict(size=100),),
}
#----------------------------------------------------------
# ----------------------------------------------------------
class SearchAdmin(admin.ModelAdmin):
list_display = ('id', 'author', 'name')
search_fields = [
'author__username',
'name',
'short_description',
'description',
'filters',
'settings',
list_display = ("id", "author", "name")
search_fields = [
"author__username",
"name",
"short_description",
"description",
"filters",
"settings",
]
list_display_links = ('id', 'name')
list_display_links = ("id", "name")
list_filter = ('author', 'name', 'version')
readonly_fields = ('hash',)
list_filter = ("author", "name", "version")
readonly_fields = ("hash",)
actions = [
rehash_search,
......@@ -109,87 +102,98 @@ class SearchAdmin(admin.ModelAdmin):
form = SearchModelForm
filter_horizontal = [
'shared_with',
'shared_with_team'
]
filter_horizontal = ["shared_with", "shared_with_team"]
fieldsets = (
(None,
dict(
fields=('name', 'author',),
),
),
('Documentation',
dict(
classes=('collapse',),
fields=('short_description', 'description',),
),
),
('Versioning',
dict(
classes=('collapse',),
fields=('version', 'previous_version', 'fork_of'),
),
),
('Sharing',
dict(
classes=('collapse',),
fields=('sharing', 'shared_with', 'shared_with_team'),
),
),
('Definition',
dict(
fields=('hash', 'filters', 'settings'),
),
),
(None, dict(fields=("name", "author",),),),
(
"Documentation",
dict(classes=("collapse",), fields=("short_description", "description",),),
),
(
"Versioning",
dict(
classes=("collapse",),
fields=("version", "previous_version", "fork_of"),
),
),
(
"Sharing",
dict(
classes=("collapse",),
fields=("sharing", "shared_with", "shared_with_team"),
),
),
("Definition", dict(fields=("hash", "filters", "settings"),),),
)
admin.site.register(Search, SearchAdmin)
class RankInline(admin.TabularInline):
model = Rank
can_delete = False
extra = 0
max_num = 0
readonly_fields = ('id', 'algorithm', 'result', 'order', 'experiment')
ordering = ('algorithm', 'order',)
model = Rank
can_delete = False
extra = 0
max_num = 0
readonly_fields = ("id", "algorithm", "result", "order", "experiment")
ordering = (
"algorithm",
"order",
)
def number_of_experiments(obj):
return obj.experiments.count()
number_of_experiments.short_description = 'Experiments'
number_of_experiments.short_description = "Experiments"
def users_to_notify(obj):
return obj.notify.count()
users_to_notify.short_description = 'Subscribed'
users_to_notify.short_description = "Subscribed"
def search_sharing(obj):
return obj.search.get_sharing_display()
search_sharing.short_description = 'Sharing'
search_sharing.short_description = "Sharing"
class LeaderboardAdmin(admin.ModelAdmin):
list_display = ('id', 'search', 'created', 'updated', users_to_notify, number_of_experiments, search_sharing)
list_display = (
"id",
"search",
"created",
"updated",
users_to_notify,
number_of_experiments,
search_sharing,
)
search_fiels = [
'search__author__username',
'search__name',
'search_short_description',
'search_description',
'filters',
'settings',
"search__author__username",
"search__name",
"search_short_description",
"search_description",
"filters",
"settings",
]
list_display_links = ('id', 'search',)
list_display_links = (
"id",
"search",
)
inlines = [
RankInline,
]
filter_horizontal = [
'notify',
"notify",
]
admin.site.register(Leaderboard, LeaderboardAdmin)
......@@ -26,46 +26,40 @@
###############################################################################
import simplejson as json
from functools import reduce
import simplejson as json
from django.conf import settings
from django.contrib.auth.models import User
from django.db.models import Q
from django.shortcuts import get_object_or_404
from rest_framework.response import Response
from rest_framework.views import APIView
from rest_framework import permissions as drf_permissions
from rest_framework import generics
from rest_framework import permissions as drf_permissions
from rest_framework import status
from rest_framework.response import Response
from rest_framework.views import APIView
from ..algorithms.models import Algorithm
from ..common import permissions as beat_permissions
from ..common.api import ShareView
from ..common.mixins import CommonContextMixin
from ..common.mixins import SerializerFieldsMixin
from ..common.models import Shareable
from ..common.responses import BadRequestResponse
from ..common.utils import ensure_html
from ..common.utils import py3_cmp
from ..databases.models import Database
from ..dataformats.models import DataFormat
from ..experiments.models import Experiment
from ..toolchains.models import Toolchain
from ..common.models import Shareable
from ..common.api import ShareView
from ..common.utils import ensure_html
from ..common.responses import BadRequestResponse
from ..common.mixins import CommonContextMixin, SerializerFieldsMixin
from ..common.utils import py3_cmp
from ..common import permissions as beat_permissions
from ..ui.templatetags.gravatar import gravatar_hash
from .utils import apply_filter
from .utils import FilterGenerator
from .utils import OR
from .models import Search
from .serializers import SearchResultSerializer, SearchSerializer, SearchWriteSerializer
from .serializers import SearchResultSerializer
from .serializers import SearchSerializer
from .serializers import SearchWriteSerializer
from .utils import OR
from .utils import FilterGenerator
from .utils import apply_filter
# ------------------------------------------------
......@@ -111,9 +105,9 @@ class SearchView(APIView):
filters = None
display_settings = None
if 'query' in data:
if not(isinstance(data['query'], str)) or (len(data['query']) == 0):
return BadRequestResponse('Invalid query data')
if "query" in data:
if not (isinstance(data["query"], str)) or (len(data["query"]) == 0):
return BadRequestResponse("Invalid query data")
query = data["query"]
else:
......
......@@ -25,14 +25,17 @@
# #
###############################################################################
from ..common.apps import CommonAppConfig
from django.utils.translation import ugettext_lazy as _
from ..common.apps import CommonAppConfig
class SearchConfig(CommonAppConfig):
name = 'beat.web.search'
verbose_name = _('Search')
name = "beat.web.search"
verbose_name = _("Search")
def ready(self):
super(SearchConfig, self).ready()
from actstream import registry
registry.register(self.get_model('Search'))
registry.register(self.get_model("Search"))
......@@ -27,5 +27,6 @@
from rest_framework import serializers
class DictListField(serializers.ListField):
child = serializers.DictField()
......@@ -28,32 +28,33 @@
import logging
logger = logging.getLogger(__name__)
from django.core.management.base import BaseCommand
from django.core.mail import EmailMessage
from django.template.loader import render_to_string
from django.conf import settings
from django.contrib.sites.models import Site
from django.core.mail import EmailMessage
from django.core.management.base import BaseCommand
from django.template.loader import render_to_string
from ...models import Leaderboard
from .... import __version__
from ...models import Leaderboard
logger = logging.getLogger(__name__)
import sys
import datetime
class Command(BaseCommand):
help = 'Update and (optionally) notify owners on leaderboard changes'
help = "Update and (optionally) notify owners on leaderboard changes"
def handle(self, *ignored, **arguments):
# Setup this command's logging level
global logger
arguments['verbosity'] = int(arguments['verbosity'])
if arguments['verbosity'] >= 1:
if arguments['verbosity'] == 1: logger.setLevel(logging.INFO)
elif arguments['verbosity'] >= 2: logger.setLevel(logging.DEBUG)
arguments["verbosity"] = int(arguments["verbosity"])
if arguments["verbosity"] >= 1:
if arguments["verbosity"] == 1:
logger.setLevel(logging.INFO)
elif arguments["verbosity"] >= 2:
logger.setLevel(logging.DEBUG)
objects_updated = 0
objects_changed = 0
......@@ -61,28 +62,32 @@ class Command(BaseCommand):
for obj in Leaderboard.objects.all():
prev_date = obj.updated
logger.info('Updating leaderboard for search %s...' % obj.search)
logger.info("Updating leaderboard for search %s..." % obj.search)
changed = obj.update_experiments()
objects_updated += 1
emails = obj.notify.values_list('email', flat=True)
emails = obj.notify.values_list("email", flat=True)
# compare tables and notify if required
if changed and emails:
logger.info('Notifying interested parties for %s' % obj.search)
logger.info("Notifying interested parties for %s" % obj.search)
objects_changed += 1
current_site = Site.objects.get_current()
template_path = 'search/leaderboard_changed.txt'
subject = "Experiment ranking for leaderboard \"%s\" changed" % obj.search.fullname()
template_path = "search/leaderboard_changed.txt"
subject = (
'Experiment ranking for leaderboard "%s" changed'
% obj.search.fullname()
)
mesg = EmailMessage(
subject.strip(),
render_to_string(template_path,
{
'leaderboard': obj,
'prev_date': prev_date,
'beat_version': __version__,
'site': current_site,
}
render_to_string(
template_path,
{
"leaderboard": obj,
"prev_date": prev_date,
"beat_version": __version__,
"site": current_site,
},
),
settings.DEFAULT_FROM_EMAIL,
to=[],
......@@ -91,7 +96,11 @@ class Command(BaseCommand):
mesg.send()
else:
logger.debug('Leaderboard %s did not change or notification is off' % obj.search)
logger.debug(
"Leaderboard %s did not change or notification is off" % obj.search
)
logger.info('Leaderboard updates: %d / Notifications: %d' % \
(objects_updated, objects_changed))
logger.info(
"Leaderboard updates: %d / Notifications: %d"
% (objects_updated, objects_changed)
)
This diff is collapsed.
......@@ -32,23 +32,32 @@ from django.db import migrations
def reset_ranks(apps, schema_editor):
'''Reset ranks before older results can be deleted'''
"""Reset ranks before older results can be deleted"""
Result = apps.get_model("experiments", "Result")
Rank = apps.get_model("search", "Rank")
total = Result.objects.count()
if total: print('')
for i, r in enumerate(Result.objects.order_by('-id')):
older = Result.objects.filter(name=r.name, id__lt=r.id,
cache=r.block.hashes.first())
if total:
print("")
for i, r in enumerate(Result.objects.order_by("-id")):
older = Result.objects.filter(
name=r.name, id__lt=r.id, cache=r.block.hashes.first()
)
for old in older:
# check if any leaderboard ranks require updates
for rank in Rank.objects.filter(result__in=(old,)):
print("Rank %d for search `%s/%s' uses old Result `%d' - " \
"resetting to newer Result `%d'..." % \
(rank.id, rank.leaderboard.search.author.username,
rank.leaderboard.search.name, old.id, r.id))
print(
"Rank %d for search `%s/%s' uses old Result `%d' - "
"resetting to newer Result `%d'..."
% (
rank.id,
rank.leaderboard.search.author.username,
rank.leaderboard.search.name,
old.id,
r.id,
)
)
rank.result.remove(old)
rank.result.add(r)
......@@ -56,8 +65,8 @@ def reset_ranks(apps, schema_editor):
class Migration(migrations.Migration):
dependencies = [
('search', '0001_initial'),
('experiments', '0002_scheduler_addons'),
("search", "0001_initial"),
("experiments", "0002_scheduler_addons"),
]
operations = [
......
This diff is collapsed.
......@@ -25,24 +25,22 @@
# #
###############################################################################
import simplejson as json
from rest_framework import serializers
from ..common.serializers import VersionableSerializer
from ..ui.templatetags.markup import restructuredtext
from .models import Search, Leaderboard
from .fields import DictListField
import simplejson as json
from .models import Leaderboard
from .models import Search
class SearchResultSerializer(serializers.Serializer):
name = serializers.SerializerMethodField()
description = serializers.CharField(source='short_description')
description = serializers.CharField(source="short_description")
def __init__(self, *args, **kwargs):
self.name_field = kwargs.pop('name_field', 'fullname')
self.name_field = kwargs.pop("name_field", "fullname")
super(SearchResultSerializer, self).__init__(*args, **kwargs)
......@@ -65,23 +63,31 @@ class SearchSerializer(VersionableSerializer):
class Meta(VersionableSerializer.Meta):
model = Search
default_fields = ['name', 'author', 'user_name', 'is_owner', 'accessibility', 'short_description', 'update_url']
default_fields = [
"name",
"author",
"user_name",
"is_owner",
"accessibility",
"short_description",
"update_url",
]
def get_user_name(self, obj):
return self.context['user'].username
return self.context["user"].username
def get_is_owner(self, obj):
return self.context['user'] == obj.author
return self.context["user"] == obj.author
def get_accessibility(self, obj):
(has_access, accessibility) = obj.accessibility_for(self.context['user'])
(has_access, accessibility) = obj.accessibility_for(self.context["user"])
return accessibility
def get_html_description(self, obj):
d = obj.description
if len(d) > 0:
return restructuredtext(d)
return ''
return ""
def get_update_url(self, obj):
return obj.get_api_update_url()
......@@ -98,53 +104,63 @@ class SearchWriteSerializer(serializers.ModelSerializer):
class Meta:
model = Search
fields = ['name', 'short_description', 'description',
'filters', 'settings', 'leaderboard']
fields = [
"name",
"short_description",
"description",
"filters",
"settings",
"leaderboard",
]
default_fields = []
def validate_name(self, name):
sanitized_name = Search.sanitize_name(name)
user = self.context['user']
user = self.context["user"]
if Search.objects.filter(author=user, name=sanitized_name).exists():
raise serializers.ValidationError('Name already used')
raise serializers.ValidationError("Name already used")
return sanitized_name
def validate(self, validated_data):
if 'description' in validated_data and\
not 'short_description' in validated_data:
raise serializers.ValidationError('Missing short description')
if (
"description" in validated_data
and "short_description" not in validated_data
):
raise serializers.ValidationError("Missing short description")
return validated_data
def create(self, validated_data):
user = self.context['user']
user = self.context["user"]
search = Search(author=user, version=1)
search.name = validated_data['name']
search.filters = json.dumps(validated_data.get('filters'))
search.settings = json.dumps(validated_data.get('settings'))
search.short_description = validated_data.get('short_description', '')
search.description = validated_data.get('description', '')
search.name = validated_data["name"]
search.filters = json.dumps(validated_data.get("filters"))
search.settings = json.dumps(validated_data.get("settings"))
search.short_description = validated_data.get("short_description", "")
search.description = validated_data.get("description", "")
search.save()
#if leaderboard activation was asked, activate the leaderboard
if validated_data.get('leaderboard'):
# if leaderboard activation was asked, activate the leaderboard
if validated_data.get("leaderboard"):
leaderboard = Leaderboard(search=search)
leaderboard.notify.add(user)
return search
def update(self, instance, validated_data):
if 'filters' in validated_data:
instance.filters = json.dumps(validated_data.get('filters'))
if 'settings' in validated_data:
instance.settings = json.dumps(validated_data.get('settings'))
instance.short_description = validated_data.get('short_description', instance.short_description)
instance.description = validated_data.get('description', instance.description)
if "filters" in validated_data:
instance.filters = json.dumps(validated_data.get("filters"))
if "settings" in validated_data:
instance.settings = json.dumps(validated_data.get("settings"))
instance.short_description = validated_data.get(
"short_description", instance.short_description
)
instance.description = validated_data.get("description", instance.description)
instance.save()
#if leaderboard activation was asked, activate the leaderboard
leaderboard = validated_data.get('leaderboard')
if leaderboard is True: #create
# if leaderboard activation was asked, activate the leaderboard
leaderboard = validated_data.get("leaderboard")
if leaderboard is True: # create
# for some reason, get_or_create always raises an IntegrityError
try:
obj = Leaderboard.objects.get(search=instance)
......@@ -152,7 +168,7 @@ class SearchWriteSerializer(serializers.ModelSerializer):
obj = Leaderboard(search=instance)
obj.save()
obj.notify.add(instance.author)
elif leaderboard is False and instance.has_leaderboard(): #delete
elif leaderboard is False and instance.has_leaderboard(): # delete
instance.leaderboard.delete()
return instance
......@@ -25,32 +25,34 @@
# #
###############################################################################
from