Skip to content
Snippets Groups Projects

Refactor update creation api

Merged Refactor update creation api
refactor_update_creation_api into master
All threads resolved!
Merged Samuel GAIST requested to merge refactor_update_creation_api into master
All threads resolved!
Viewing commit 672d7678
Prev Next
Show latest version
1 file
+ 332
− 225
Compare changes
  • Side-by-side
  • Inline
beat/web/search/api.py
+ 332
− 225
@@ -25,6 +25,11 @@
# #
###############################################################################
import simplejson as json
from functools import reduce
from django.conf import settings
from django.contrib.auth.models import User
from django.db.models import Q
@@ -37,33 +42,32 @@ from rest_framework import permissions
from rest_framework import generics
from rest_framework import status
from .utils import apply_filter
from .utils import FilterGenerator
from .utils import OR
from ..algorithms.models import Algorithm
from ..databases.models import Database
from ..dataformats.models import DataFormat
from ..experiments.models import Experiment
from ..toolchains.models import Toolchain
from ..common.models import Shareable
from ..common.mixins import IsAuthorOrReadOnlyMixin
from ..common.api import ShareView
from ..common.utils import ensure_html
from .models import Search
from ..common.responses import BadRequestResponse
from ..common.mixins import CommonContextMixin, SerializerFieldsMixin
from ..common.utils import py3_cmp
from ..ui.templatetags.gravatar import gravatar_hash
from .serializers import SearchResultSerializer, SearchSerializer, SearchWriteSerializer
from .utils import apply_filter
from .utils import FilterGenerator
from .utils import OR
import simplejson as json
from .models import Search
from .serializers import SearchResultSerializer, SearchSerializer, SearchWriteSerializer
#------------------------------------------------
# ------------------------------------------------
class SearchView(APIView):
@@ -81,19 +85,24 @@ class SearchView(APIView):
'order-by'
"""
permission_classes = [permissions.AllowAny]
permission_classes = [permissions.AllowAny]
FILTER_IEXACT = 0
FILTER_ICONTAINS = 1
FILTER_IEXACT = 0
FILTER_ICONTAINS = 1
FILTER_ISTARTSWITH = 2
FILTER_IENDSWITH = 3
FILTER_IENDSWITH = 3
@staticmethod
def build_name_and_description_query(keywords):
return reduce(lambda a, b: a & b, map(lambda keyword: Q(name__icontains=keyword) |
Q(short_description__icontains=keyword), keywords))
return reduce(
lambda a, b: a & b,
map(
lambda keyword: Q(name__icontains=keyword)
| Q(short_description__icontains=keyword),
keywords,
),
)
def post(self, request):
data = request.data
@@ -102,225 +111,289 @@ class SearchView(APIView):
filters = None
display_settings = None
if 'query' in data:
if not(isinstance(data['query'], six.string_types)) or \
(len(data['query']) == 0):
return BadRequestResponse('Invalid query data')
if "query" in data:
if not (isinstance(data["query"], six.string_types)) or (
len(data["query"]) == 0
):
return BadRequestResponse("Invalid query data")
query = data['query']
query = data["query"]
else:
if not(isinstance(data['filters'], list)) or (len(data['filters']) == 0):
return BadRequestResponse('Invalid filter data')
if not (isinstance(data["filters"], list)) or (len(data["filters"]) == 0):
return BadRequestResponse("Invalid filter data")
filters = data['filters']
if 'settings' in data:
display_settings = data['settings']
filters = data["filters"]
if "settings" in data:
display_settings = data["settings"]
# Process the query
scope_database = None
scope_type = None
scope_database = None
scope_type = None
scope_toolchain = None
scope_algorithm = None
scope_analyzer = None
keywords = []
scope_analyzer = None
keywords = []
if filters is None:
for keyword in map(lambda x: x.strip(), query.split(' ')):
offset = keyword.find(':')
for keyword in map(lambda x: x.strip(), query.split(" ")):
offset = keyword.find(":")
if offset != -1:
command = keyword[:offset]
keyword = keyword[offset+1:]
if command in ['db', 'database']:
scope_database = keyword.split(',')
elif command in ['tc', 'toolchain']:
scope_toolchain = keyword.split(',')
elif command in ['algo', 'algorithm']:
scope_algorithm = keyword.split(',')
elif command == 'analyzer':
scope_analyzer = keyword.split(',')
elif command == 'type':
if keyword in ['results', 'toolchains', 'algorithms', 'analyzers',
'dataformats', 'databases', 'users']:
keyword = keyword[offset + 1 :]
if command in ["db", "database"]:
scope_database = keyword.split(",")
elif command in ["tc", "toolchain"]:
scope_toolchain = keyword.split(",")
elif command in ["algo", "algorithm"]:
scope_algorithm = keyword.split(",")
elif command == "analyzer":
scope_analyzer = keyword.split(",")
elif command == "type":
if keyword in [
"results",
"toolchains",
"algorithms",
"analyzers",
"dataformats",
"databases",
"users",
]:
scope_type = keyword
else:
keywords.append(keyword)
if (scope_type is None) or (scope_type == 'results'):
if (scope_type is None) or (scope_type == "results"):
filters = []
if scope_toolchain is not None:
if len(scope_toolchain) > 1:
filters.append({
'context': 'toolchain',
'name': None,
'operator': 'contains-any-of',
'value': scope_toolchain,
})
filters.append(
{
"context": "toolchain",
"name": None,
"operator": "contains-any-of",
"value": scope_toolchain,
}
)
elif len(scope_toolchain) == 1:
filters.append({
'context': 'toolchain',
'name': None,
'operator': 'contains',
'value': scope_toolchain[0],
})
filters.append(
{
"context": "toolchain",
"name": None,
"operator": "contains",
"value": scope_toolchain[0],
}
)
if scope_algorithm is not None:
if len(scope_algorithm) > 1:
filters.append({
'context': 'algorithm',
'name': None,
'operator': 'contains-any-of',
'value': scope_algorithm,
})
filters.append(
{
"context": "algorithm",
"name": None,
"operator": "contains-any-of",
"value": scope_algorithm,
}
)
elif len(scope_algorithm) == 1:
filters.append({
'context': 'algorithm',
'name': None,
'operator': 'contains',
'value': scope_algorithm[0],
})
filters.append(
{
"context": "algorithm",
"name": None,
"operator": "contains",
"value": scope_algorithm[0],
}
)
if scope_analyzer is not None:
if len(scope_analyzer) > 1:
filters.append({
'context': 'analyzer',
'name': None,
'operator': 'contains-any-of',
'value': scope_analyzer,
})
filters.append(
{
"context": "analyzer",
"name": None,
"operator": "contains-any-of",
"value": scope_analyzer,
}
)
elif len(scope_analyzer) == 1:
filters.append({
'context': 'analyzer',
'name': None,
'operator': 'contains',
'value': scope_analyzer[0],
})
filters.append(
{
"context": "analyzer",
"name": None,
"operator": "contains",
"value": scope_analyzer[0],
}
)
if scope_database is not None:
if len(scope_database) > 1:
filters.append({
'context': 'database-name',
'name': None,
'operator': 'contains-any-of',
'value': scope_database,
})
filters.append(
{
"context": "database-name",
"name": None,
"operator": "contains-any-of",
"value": scope_database,
}
)
elif len(scope_database) == 1:
filters.append({
'context': 'database-name',
'name': None,
'operator': 'contains',
'value': scope_database[0],
})
filters.append(
{
"context": "database-name",
"name": None,
"operator": "contains",
"value": scope_database[0],
}
)
if len(keywords) > 0:
filters.append({
'context': 'any-field',
'name': None,
'operator': 'contains-any-of',
'value': keywords,
})
filters.append(
{
"context": "any-field",
"name": None,
"operator": "contains-any-of",
"value": keywords,
}
)
else:
scope_type = 'results'
scope_type = "results"
result = {
'users': [],
'toolchains': [],
'algorithms': [],
'analyzers': [],
'dataformats': [],
'databases': [],
'results': [],
'filters': filters,
'settings': display_settings,
'query': {
'type': scope_type,
},
"users": [],
"toolchains": [],
"algorithms": [],
"analyzers": [],
"dataformats": [],
"databases": [],
"results": [],
"filters": filters,
"settings": display_settings,
"query": {"type": scope_type},
}
# Search for users matching the query
if (scope_database is None) and (scope_toolchain is None) and \
(scope_algorithm is None) and (scope_analyzer is None) and \
((scope_type is None) or (scope_type == 'users')):
result['users'] = []
if (
(scope_database is None)
and (scope_toolchain is None)
and (scope_algorithm is None)
and (scope_analyzer is None)
and ((scope_type is None) or (scope_type == "users"))
):
result["users"] = []
if len(keywords) > 0:
q = reduce(lambda a, b: a & b, map(lambda keyword: Q(username__icontains=keyword), keywords))
users = User.objects.filter(q).exclude(username__in=settings.ACCOUNTS_TO_EXCLUDE_FROM_SEARCH).order_by('username')
result['users'] = map(lambda u: { 'username': u.username,
'gravatar_hash': gravatar_hash(u.email),
'join_date': u.date_joined.strftime('%b %d, %Y')
}, users)
q = reduce(
lambda a, b: a & b,
map(lambda keyword: Q(username__icontains=keyword), keywords),
)
users = (
User.objects.filter(q)
.exclude(username__in=settings.ACCOUNTS_TO_EXCLUDE_FROM_SEARCH)
.order_by("username")
)
result["users"] = map(
lambda u: {
"username": u.username,
"gravatar_hash": gravatar_hash(u.email),
"join_date": u.date_joined.strftime("%b %d, %Y"),
},
users,
)
query = None
if len(keywords) > 0:
query = self.build_name_and_description_query(keywords)
# Search for toolchains matching the query
if (scope_database is None) and (scope_algorithm is None) and \
(scope_analyzer is None) and ((scope_type is None) or (scope_type == 'toolchains')):
result['toolchains'] = self._retrieve_contributions(
Toolchain.objects.for_user(request.user, True),
scope_toolchain, query
if (
(scope_database is None)
and (scope_algorithm is None)
and (scope_analyzer is None)
and ((scope_type is None) or (scope_type == "toolchains"))
):
result["toolchains"] = self._retrieve_contributions(
Toolchain.objects.for_user(request.user, True), scope_toolchain, query
)
# Search for algorithms matching the query
if (scope_database is None) and (scope_toolchain is None) and \
(scope_analyzer is None) and ((scope_type is None) or (scope_type == 'algorithms')):
result['algorithms'] = self._retrieve_contributions(
Algorithm.objects.for_user(request.user, True).filter(result_dataformat__isnull=True),
scope_algorithm, query
if (
(scope_database is None)
and (scope_toolchain is None)
and (scope_analyzer is None)
and ((scope_type is None) or (scope_type == "algorithms"))
):
result["algorithms"] = self._retrieve_contributions(
Algorithm.objects.for_user(request.user, True).filter(
result_dataformat__isnull=True
),
scope_algorithm,
query,
)
# Search for analyzers matching the query
if (scope_database is None) and (scope_toolchain is None) and \
(scope_algorithm is None) and ((scope_type is None) or (scope_type == 'analyzers')):
result['analyzers'] = self._retrieve_contributions(
Algorithm.objects.for_user(request.user, True).filter(result_dataformat__isnull=False),
scope_analyzer, query
if (
(scope_database is None)
and (scope_toolchain is None)
and (scope_algorithm is None)
and ((scope_type is None) or (scope_type == "analyzers"))
):
result["analyzers"] = self._retrieve_contributions(
Algorithm.objects.for_user(request.user, True).filter(
result_dataformat__isnull=False
),
scope_analyzer,
query,
)
# Search for data formats matching the query
if (scope_database is None) and (scope_toolchain is None) and \
(scope_algorithm is None) and (scope_analyzer is None) and \
((scope_type is None) or (scope_type == 'dataformats')):
if (
(scope_database is None)
and (scope_toolchain is None)
and (scope_algorithm is None)
and (scope_analyzer is None)
and ((scope_type is None) or (scope_type == "dataformats"))
):
dataformats = DataFormat.objects.for_user(request.user, True)
if query:
dataformats = dataformats.filter(query)
serializer = SearchResultSerializer(dataformats, many=True)
result['dataformats'] = serializer.data
result["dataformats"] = serializer.data
# Search for databases matching the query
if (scope_toolchain is None) and (scope_algorithm is None) and \
(scope_analyzer is None) and ((scope_type is None) or (scope_type == 'databases')):
result['databases'] = self._retrieve_databases(Database.objects.for_user(request.user, True), scope_database, query)
if (
(scope_toolchain is None)
and (scope_algorithm is None)
and (scope_analyzer is None)
and ((scope_type is None) or (scope_type == "databases"))
):
result["databases"] = self._retrieve_databases(
Database.objects.for_user(request.user, True), scope_database, query
)
# Search for experiments matching the query
if ((scope_type is None) or (scope_type == 'results')):
result['results'] = self._retrieve_experiment_results(request.user, filters)
if (scope_type is None) or (scope_type == "results"):
result["results"] = self._retrieve_experiment_results(request.user, filters)
# Sort the results
result['toolchains'].sort(lambda x, y: cmp(x['name'], y['name']))
result['algorithms'].sort(lambda x, y: cmp(x['name'], y['name']))
result['analyzers'].sort(lambda x, y: cmp(x['name'], y['name']))
result['dataformats'].sort(lambda x, y: cmp(x['name'], y['name']))
result['databases'].sort(lambda x, y: cmp(x['name'], y['name']))
result["toolchains"].sort(lambda x, y: py3_cmp(x["name"], y["name"]))
result["algorithms"].sort(lambda x, y: py3_cmp(x["name"], y["name"]))
result["analyzers"].sort(lambda x, y: py3_cmp(x["name"], y["name"]))
result["dataformats"].sort(lambda x, y: py3_cmp(x["name"], y["name"]))
result["databases"].sort(lambda x, y: py3_cmp(x["name"], y["name"]))
return Response(result)
def _retrieve_contributions(self, queryset, scope, query):
generator = FilterGenerator()
scope_filters = []
if scope is not None:
for contribution_name in scope:
scope_filters.append(generator.process_contribution_name(contribution_name))
scope_filters.append(
generator.process_contribution_name(contribution_name)
)
if len(scope_filters):
queryset = queryset.filter(OR(scope_filters))
@@ -331,7 +404,6 @@ class SearchView(APIView):
serializer = SearchResultSerializer(queryset, many=True)
return serializer.data
def _retrieve_databases(self, queryset, scope, query):
generator = FilterGenerator()
@@ -346,26 +418,26 @@ class SearchView(APIView):
if query:
queryset = queryset.filter(query)
queryset= queryset.distinct()
queryset = queryset.distinct()
serializer = SearchResultSerializer(queryset, many=True, name_field='name')
serializer = SearchResultSerializer(queryset, many=True, name_field="name")
return serializer.data
def _retrieve_experiment_results(self, user, filters):
results = {
'experiments': [],
'dataformats': {},
'common_analyzers': [],
'common_protocols': [],
"experiments": [],
"dataformats": {},
"common_analyzers": [],
"common_protocols": [],
}
if len(filters) == 0:
return results
# Use the experiment filters
experiments = Experiment.objects.for_user(user, True).filter(status=Experiment.DONE)
experiments = Experiment.objects.for_user(user, True).filter(
status=Experiment.DONE
)
for filter_entry in filters:
experiments = apply_filter(experiments, filter_entry)
@@ -375,7 +447,6 @@ class SearchView(APIView):
if experiments.count() == 0:
return results
# Retrieve informations about each experiment and determine if there is at least
# one common analyzer
common_protocols = None
@@ -383,77 +454,95 @@ class SearchView(APIView):
for experiment in experiments.iterator():
experiment_entry = {
'name': experiment.fullname(),
'toolchain': experiment.toolchain.fullname(),
'description': experiment.short_description,
'public': (experiment.sharing == Shareable.PUBLIC),
'attestation_number': None,
'attestation_locked': False,
'end_date': experiment.end_date,
'protocols': list(set(map(lambda x: x.protocol.fullname(), experiment.referenced_datasets.iterator()))),
'analyzers': [],
"name": experiment.fullname(),
"toolchain": experiment.toolchain.fullname(),
"description": experiment.short_description,
"public": (experiment.sharing == Shareable.PUBLIC),
"attestation_number": None,
"attestation_locked": False,
"end_date": experiment.end_date,
"protocols": list(
set(
map(
lambda x: x.protocol.fullname(),
experiment.referenced_datasets.iterator(),
)
)
),
"analyzers": [],
}
if experiment.has_attestation():
experiment_entry['attestation_number'] = experiment.attestation.number
experiment_entry['attestation_locked'] = experiment.attestation.locked
experiment_entry["attestation_number"] = experiment.attestation.number
experiment_entry["attestation_locked"] = experiment.attestation.locked
experiment_analyzers = []
for analyzer_block in experiment.blocks.filter(analyzer=True).iterator():
analyzer_entry = {
'name': analyzer_block.algorithm.fullname(),
'block': analyzer_block.name,
'results': {},
"name": analyzer_block.algorithm.fullname(),
"block": analyzer_block.name,
"results": {},
}
experiment_entry['analyzers'].append(analyzer_entry)
experiment_analyzers.append(analyzer_entry['name'])
experiment_entry["analyzers"].append(analyzer_entry)
experiment_analyzers.append(analyzer_entry["name"])
if analyzer_entry['name'] not in results['dataformats']:
results['dataformats'][analyzer_entry['name']] = json.loads(analyzer_block.algorithm.result_dataformat)
if analyzer_entry["name"] not in results["dataformats"]:
results["dataformats"][analyzer_entry["name"]] = json.loads(
analyzer_block.algorithm.result_dataformat
)
if common_analyzers is None:
common_analyzers = experiment_analyzers
elif len(common_analyzers) > 0:
common_analyzers = filter(lambda x: x in experiment_analyzers, common_analyzers)
common_analyzers = filter(
lambda x: x in experiment_analyzers, common_analyzers
)
if common_protocols is None:
common_protocols = experiment_entry['protocols']
common_protocols = experiment_entry["protocols"]
elif len(common_protocols) > 0:
common_protocols = filter(lambda x: x in experiment_entry['protocols'], common_protocols)
common_protocols = filter(
lambda x: x in experiment_entry["protocols"], common_protocols
)
results['experiments'].append(experiment_entry)
results['common_analyzers'] = common_analyzers
results['common_protocols'] = common_protocols
results["experiments"].append(experiment_entry)
results["common_analyzers"] = common_analyzers
results["common_protocols"] = common_protocols
# No common analyzer found, don't retrieve any result
if len(common_analyzers) == 0:
results['dataformats'] = {}
results["dataformats"] = {}
return results
# Retrieve the results of each experiment
for index, experiment in enumerate(experiments.iterator()):
for analyzer_block in experiment.blocks.filter(analyzer=True).iterator():
analyzer_entry = filter(lambda x: x['block'] == analyzer_block.name,
results['experiments'][index]['analyzers'])[0]
analyzer_entry = filter(
lambda x: x["block"] == analyzer_block.name,
results["experiments"][index]["analyzers"],
)[0]
for analyzer_result in analyzer_block.results.iterator():
analyzer_entry['results'][analyzer_result.name] = {
'type': analyzer_result.type,
'primary': analyzer_result.primary,
'value': analyzer_result.value()
analyzer_entry["results"][analyzer_result.name] = {
"type": analyzer_result.type,
"primary": analyzer_result.primary,
"value": analyzer_result.value(),
}
return results
#------------------------------------------------
# ------------------------------------------------
class SearchSaveView(CommonContextMixin, SerializerFieldsMixin, generics.CreateAPIView, generics.UpdateAPIView):
class SearchSaveView(
CommonContextMixin,
SerializerFieldsMixin,
generics.CreateAPIView,
generics.UpdateAPIView,
):
"""
This endpoint allows to save and update a search query
@@ -474,12 +563,12 @@ class SearchSaveView(CommonContextMixin, SerializerFieldsMixin, generics.CreateA
fields_to_return = self.get_serializer_fields(request)
# Retrieve the description in HTML format
if 'html_description' in fields_to_return:
if "html_description" in fields_to_return:
description = search.description
if len(description) > 0:
result['html_description'] = ensure_html(description)
result["html_description"] = ensure_html(description)
else:
result['html_description'] = ''
result["html_description"] = ""
return result
def post(self, request):
@@ -487,73 +576,91 @@ class SearchSaveView(CommonContextMixin, SerializerFieldsMixin, generics.CreateA
serializer.is_valid(raise_exception=True)
search = serializer.save()
result = self.build_results(request, search)
result['fullname'] = search.fullname()
result['url'] = search.get_absolute_url()
result["fullname"] = search.fullname()
result["url"] = search.get_absolute_url()
return Response(result, status=status.HTTP_201_CREATED)
def put(self, request, author_name, name):
search = get_object_or_404(Search, author__username=author_name, name=name)
serializer = self.get_serializer(instance=search, data=request.data, partial=True)
serializer = self.get_serializer(
instance=search, data=request.data, partial=True
)
serializer.is_valid(raise_exception=True)
serializer.save()
result = self.build_results(request, search)
return Response(result)
#------------------------------------------------
# ------------------------------------------------
class ListSearchView(CommonContextMixin, generics.ListAPIView):
"""
Lists all available search from a user
"""
permission_classes = [permissions.AllowAny]
serializer_class = SearchSerializer
def get_queryset(self):
author_name = self.kwargs['author_name']
return Search.objects.for_user(self.request.user, True).select_related().filter(author__username=author_name)
author_name = self.kwargs["author_name"]
return (
Search.objects.for_user(self.request.user, True)
.select_related()
.filter(author__username=author_name)
)
#----------------------------------------------------------
# ----------------------------------------------------------
class RetrieveDestroySearchAPIView(CommonContextMixin, SerializerFieldsMixin, IsAuthorOrReadOnlyMixin, generics.RetrieveDestroyAPIView):
class RetrieveDestroySearchAPIView(
CommonContextMixin,
SerializerFieldsMixin,
IsAuthorOrReadOnlyMixin,
generics.RetrieveDestroyAPIView,
):
"""
Delete the given search
"""
model = Search
serializer_class = SearchSerializer
def get_object(self):
author_name = self.kwargs.get('author_name')
name = self.kwargs.get('object_name')
author_name = self.kwargs.get("author_name")
name = self.kwargs.get("object_name")
user = self.request.user
return get_object_or_404(self.model.objects.for_user(user, True),
author__username=author_name,
name=name)
return get_object_or_404(
self.model.objects.for_user(user, True),
author__username=author_name,
name=name,
)
def get(self, request, *args, **kwargs):
search = self.get_object()
# Process the query string
allow_sharing = request.user == search.author
fields_to_return = self.get_serializer_fields(request, allow_sharing=allow_sharing)
fields_to_return = self.get_serializer_fields(
request, allow_sharing=allow_sharing
)
serializer = self.get_serializer(search, fields=fields_to_return)
return Response(serializer.data)
#------------------------------------------------
# ------------------------------------------------
class ShareSearchView(ShareView):
"""
Share the given search with other users/teams
"""
model = Search
permission_classes = [permissions.AllowAny]
def get_queryset(self):
self.kwargs['version'] = 1
self.kwargs["version"] = 1
return super(ShareSearchView, self).get_queryset()