diff --git a/beat/web/search/api.py b/beat/web/search/api.py index 3f1c6f5039ea654863e82d43840d80fb85201437..01f005e27612c5a7c5dba121be3503a9f9e1f5a1 100644 --- a/beat/web/search/api.py +++ b/beat/web/search/api.py @@ -25,6 +25,11 @@ # # ############################################################################### + +import simplejson as json + +from functools import reduce + from django.conf import settings from django.contrib.auth.models import User from django.db.models import Q @@ -37,33 +42,32 @@ from rest_framework import permissions from rest_framework import generics from rest_framework import status -from .utils import apply_filter -from .utils import FilterGenerator -from .utils import OR - from ..algorithms.models import Algorithm from ..databases.models import Database from ..dataformats.models import DataFormat from ..experiments.models import Experiment from ..toolchains.models import Toolchain + from ..common.models import Shareable from ..common.mixins import IsAuthorOrReadOnlyMixin from ..common.api import ShareView from ..common.utils import ensure_html - -from .models import Search - from ..common.responses import BadRequestResponse from ..common.mixins import CommonContextMixin, SerializerFieldsMixin +from ..common.utils import py3_cmp from ..ui.templatetags.gravatar import gravatar_hash -from .serializers import SearchResultSerializer, SearchSerializer, SearchWriteSerializer +from .utils import apply_filter +from .utils import FilterGenerator +from .utils import OR -import simplejson as json +from .models import Search + +from .serializers import SearchResultSerializer, SearchSerializer, SearchWriteSerializer -#------------------------------------------------ +# ------------------------------------------------ class SearchView(APIView): @@ -81,19 +85,24 @@ class SearchView(APIView): 'order-by' """ - permission_classes = [permissions.AllowAny] + permission_classes = [permissions.AllowAny] - FILTER_IEXACT = 0 - FILTER_ICONTAINS = 1 + FILTER_IEXACT = 0 + FILTER_ICONTAINS = 1 FILTER_ISTARTSWITH = 2 - FILTER_IENDSWITH = 3 - + FILTER_IENDSWITH = 3 @staticmethod def build_name_and_description_query(keywords): - return reduce(lambda a, b: a & b, map(lambda keyword: Q(name__icontains=keyword) | - Q(short_description__icontains=keyword), keywords)) + return reduce( + lambda a, b: a & b, + map( + lambda keyword: Q(name__icontains=keyword) + | Q(short_description__icontains=keyword), + keywords, + ), + ) def post(self, request): data = request.data @@ -102,225 +111,289 @@ class SearchView(APIView): filters = None display_settings = None - if 'query' in data: - if not(isinstance(data['query'], six.string_types)) or \ - (len(data['query']) == 0): - return BadRequestResponse('Invalid query data') + if "query" in data: + if not (isinstance(data["query"], six.string_types)) or ( + len(data["query"]) == 0 + ): + return BadRequestResponse("Invalid query data") - query = data['query'] + query = data["query"] else: - if not(isinstance(data['filters'], list)) or (len(data['filters']) == 0): - return BadRequestResponse('Invalid filter data') + if not (isinstance(data["filters"], list)) or (len(data["filters"]) == 0): + return BadRequestResponse("Invalid filter data") - filters = data['filters'] - - if 'settings' in data: - display_settings = data['settings'] + filters = data["filters"] + if "settings" in data: + display_settings = data["settings"] # Process the query - scope_database = None - scope_type = None + scope_database = None + scope_type = None scope_toolchain = None scope_algorithm = None - scope_analyzer = None - keywords = [] + scope_analyzer = None + keywords = [] if filters is None: - for keyword in map(lambda x: x.strip(), query.split(' ')): - offset = keyword.find(':') + for keyword in map(lambda x: x.strip(), query.split(" ")): + offset = keyword.find(":") if offset != -1: command = keyword[:offset] - keyword = keyword[offset+1:] - - if command in ['db', 'database']: - scope_database = keyword.split(',') - elif command in ['tc', 'toolchain']: - scope_toolchain = keyword.split(',') - elif command in ['algo', 'algorithm']: - scope_algorithm = keyword.split(',') - elif command == 'analyzer': - scope_analyzer = keyword.split(',') - elif command == 'type': - if keyword in ['results', 'toolchains', 'algorithms', 'analyzers', - 'dataformats', 'databases', 'users']: + keyword = keyword[offset + 1 :] + + if command in ["db", "database"]: + scope_database = keyword.split(",") + elif command in ["tc", "toolchain"]: + scope_toolchain = keyword.split(",") + elif command in ["algo", "algorithm"]: + scope_algorithm = keyword.split(",") + elif command == "analyzer": + scope_analyzer = keyword.split(",") + elif command == "type": + if keyword in [ + "results", + "toolchains", + "algorithms", + "analyzers", + "dataformats", + "databases", + "users", + ]: scope_type = keyword else: keywords.append(keyword) - if (scope_type is None) or (scope_type == 'results'): + if (scope_type is None) or (scope_type == "results"): filters = [] if scope_toolchain is not None: if len(scope_toolchain) > 1: - filters.append({ - 'context': 'toolchain', - 'name': None, - 'operator': 'contains-any-of', - 'value': scope_toolchain, - }) + filters.append( + { + "context": "toolchain", + "name": None, + "operator": "contains-any-of", + "value": scope_toolchain, + } + ) elif len(scope_toolchain) == 1: - filters.append({ - 'context': 'toolchain', - 'name': None, - 'operator': 'contains', - 'value': scope_toolchain[0], - }) + filters.append( + { + "context": "toolchain", + "name": None, + "operator": "contains", + "value": scope_toolchain[0], + } + ) if scope_algorithm is not None: if len(scope_algorithm) > 1: - filters.append({ - 'context': 'algorithm', - 'name': None, - 'operator': 'contains-any-of', - 'value': scope_algorithm, - }) + filters.append( + { + "context": "algorithm", + "name": None, + "operator": "contains-any-of", + "value": scope_algorithm, + } + ) elif len(scope_algorithm) == 1: - filters.append({ - 'context': 'algorithm', - 'name': None, - 'operator': 'contains', - 'value': scope_algorithm[0], - }) + filters.append( + { + "context": "algorithm", + "name": None, + "operator": "contains", + "value": scope_algorithm[0], + } + ) if scope_analyzer is not None: if len(scope_analyzer) > 1: - filters.append({ - 'context': 'analyzer', - 'name': None, - 'operator': 'contains-any-of', - 'value': scope_analyzer, - }) + filters.append( + { + "context": "analyzer", + "name": None, + "operator": "contains-any-of", + "value": scope_analyzer, + } + ) elif len(scope_analyzer) == 1: - filters.append({ - 'context': 'analyzer', - 'name': None, - 'operator': 'contains', - 'value': scope_analyzer[0], - }) + filters.append( + { + "context": "analyzer", + "name": None, + "operator": "contains", + "value": scope_analyzer[0], + } + ) if scope_database is not None: if len(scope_database) > 1: - filters.append({ - 'context': 'database-name', - 'name': None, - 'operator': 'contains-any-of', - 'value': scope_database, - }) + filters.append( + { + "context": "database-name", + "name": None, + "operator": "contains-any-of", + "value": scope_database, + } + ) elif len(scope_database) == 1: - filters.append({ - 'context': 'database-name', - 'name': None, - 'operator': 'contains', - 'value': scope_database[0], - }) + filters.append( + { + "context": "database-name", + "name": None, + "operator": "contains", + "value": scope_database[0], + } + ) if len(keywords) > 0: - filters.append({ - 'context': 'any-field', - 'name': None, - 'operator': 'contains-any-of', - 'value': keywords, - }) + filters.append( + { + "context": "any-field", + "name": None, + "operator": "contains-any-of", + "value": keywords, + } + ) else: - scope_type = 'results' - + scope_type = "results" result = { - 'users': [], - 'toolchains': [], - 'algorithms': [], - 'analyzers': [], - 'dataformats': [], - 'databases': [], - 'results': [], - 'filters': filters, - 'settings': display_settings, - 'query': { - 'type': scope_type, - }, + "users": [], + "toolchains": [], + "algorithms": [], + "analyzers": [], + "dataformats": [], + "databases": [], + "results": [], + "filters": filters, + "settings": display_settings, + "query": {"type": scope_type}, } - # Search for users matching the query - if (scope_database is None) and (scope_toolchain is None) and \ - (scope_algorithm is None) and (scope_analyzer is None) and \ - ((scope_type is None) or (scope_type == 'users')): - result['users'] = [] + if ( + (scope_database is None) + and (scope_toolchain is None) + and (scope_algorithm is None) + and (scope_analyzer is None) + and ((scope_type is None) or (scope_type == "users")) + ): + result["users"] = [] if len(keywords) > 0: - q = reduce(lambda a, b: a & b, map(lambda keyword: Q(username__icontains=keyword), keywords)) - users = User.objects.filter(q).exclude(username__in=settings.ACCOUNTS_TO_EXCLUDE_FROM_SEARCH).order_by('username') - - result['users'] = map(lambda u: { 'username': u.username, - 'gravatar_hash': gravatar_hash(u.email), - 'join_date': u.date_joined.strftime('%b %d, %Y') - }, users) + q = reduce( + lambda a, b: a & b, + map(lambda keyword: Q(username__icontains=keyword), keywords), + ) + users = ( + User.objects.filter(q) + .exclude(username__in=settings.ACCOUNTS_TO_EXCLUDE_FROM_SEARCH) + .order_by("username") + ) + + result["users"] = map( + lambda u: { + "username": u.username, + "gravatar_hash": gravatar_hash(u.email), + "join_date": u.date_joined.strftime("%b %d, %Y"), + }, + users, + ) query = None if len(keywords) > 0: query = self.build_name_and_description_query(keywords) # Search for toolchains matching the query - if (scope_database is None) and (scope_algorithm is None) and \ - (scope_analyzer is None) and ((scope_type is None) or (scope_type == 'toolchains')): - result['toolchains'] = self._retrieve_contributions( - Toolchain.objects.for_user(request.user, True), - scope_toolchain, query + if ( + (scope_database is None) + and (scope_algorithm is None) + and (scope_analyzer is None) + and ((scope_type is None) or (scope_type == "toolchains")) + ): + result["toolchains"] = self._retrieve_contributions( + Toolchain.objects.for_user(request.user, True), scope_toolchain, query ) # Search for algorithms matching the query - if (scope_database is None) and (scope_toolchain is None) and \ - (scope_analyzer is None) and ((scope_type is None) or (scope_type == 'algorithms')): - result['algorithms'] = self._retrieve_contributions( - Algorithm.objects.for_user(request.user, True).filter(result_dataformat__isnull=True), - scope_algorithm, query + if ( + (scope_database is None) + and (scope_toolchain is None) + and (scope_analyzer is None) + and ((scope_type is None) or (scope_type == "algorithms")) + ): + result["algorithms"] = self._retrieve_contributions( + Algorithm.objects.for_user(request.user, True).filter( + result_dataformat__isnull=True + ), + scope_algorithm, + query, ) # Search for analyzers matching the query - if (scope_database is None) and (scope_toolchain is None) and \ - (scope_algorithm is None) and ((scope_type is None) or (scope_type == 'analyzers')): - result['analyzers'] = self._retrieve_contributions( - Algorithm.objects.for_user(request.user, True).filter(result_dataformat__isnull=False), - scope_analyzer, query + if ( + (scope_database is None) + and (scope_toolchain is None) + and (scope_algorithm is None) + and ((scope_type is None) or (scope_type == "analyzers")) + ): + result["analyzers"] = self._retrieve_contributions( + Algorithm.objects.for_user(request.user, True).filter( + result_dataformat__isnull=False + ), + scope_analyzer, + query, ) # Search for data formats matching the query - if (scope_database is None) and (scope_toolchain is None) and \ - (scope_algorithm is None) and (scope_analyzer is None) and \ - ((scope_type is None) or (scope_type == 'dataformats')): + if ( + (scope_database is None) + and (scope_toolchain is None) + and (scope_algorithm is None) + and (scope_analyzer is None) + and ((scope_type is None) or (scope_type == "dataformats")) + ): dataformats = DataFormat.objects.for_user(request.user, True) if query: dataformats = dataformats.filter(query) serializer = SearchResultSerializer(dataformats, many=True) - result['dataformats'] = serializer.data + result["dataformats"] = serializer.data # Search for databases matching the query - if (scope_toolchain is None) and (scope_algorithm is None) and \ - (scope_analyzer is None) and ((scope_type is None) or (scope_type == 'databases')): - result['databases'] = self._retrieve_databases(Database.objects.for_user(request.user, True), scope_database, query) + if ( + (scope_toolchain is None) + and (scope_algorithm is None) + and (scope_analyzer is None) + and ((scope_type is None) or (scope_type == "databases")) + ): + result["databases"] = self._retrieve_databases( + Database.objects.for_user(request.user, True), scope_database, query + ) # Search for experiments matching the query - if ((scope_type is None) or (scope_type == 'results')): - result['results'] = self._retrieve_experiment_results(request.user, filters) + if (scope_type is None) or (scope_type == "results"): + result["results"] = self._retrieve_experiment_results(request.user, filters) # Sort the results - result['toolchains'].sort(lambda x, y: cmp(x['name'], y['name'])) - result['algorithms'].sort(lambda x, y: cmp(x['name'], y['name'])) - result['analyzers'].sort(lambda x, y: cmp(x['name'], y['name'])) - result['dataformats'].sort(lambda x, y: cmp(x['name'], y['name'])) - result['databases'].sort(lambda x, y: cmp(x['name'], y['name'])) + result["toolchains"].sort(lambda x, y: py3_cmp(x["name"], y["name"])) + result["algorithms"].sort(lambda x, y: py3_cmp(x["name"], y["name"])) + result["analyzers"].sort(lambda x, y: py3_cmp(x["name"], y["name"])) + result["dataformats"].sort(lambda x, y: py3_cmp(x["name"], y["name"])) + result["databases"].sort(lambda x, y: py3_cmp(x["name"], y["name"])) return Response(result) - def _retrieve_contributions(self, queryset, scope, query): generator = FilterGenerator() scope_filters = [] if scope is not None: for contribution_name in scope: - scope_filters.append(generator.process_contribution_name(contribution_name)) + scope_filters.append( + generator.process_contribution_name(contribution_name) + ) if len(scope_filters): queryset = queryset.filter(OR(scope_filters)) @@ -331,7 +404,6 @@ class SearchView(APIView): serializer = SearchResultSerializer(queryset, many=True) return serializer.data - def _retrieve_databases(self, queryset, scope, query): generator = FilterGenerator() @@ -346,26 +418,26 @@ class SearchView(APIView): if query: queryset = queryset.filter(query) - queryset= queryset.distinct() + queryset = queryset.distinct() - serializer = SearchResultSerializer(queryset, many=True, name_field='name') + serializer = SearchResultSerializer(queryset, many=True, name_field="name") return serializer.data - def _retrieve_experiment_results(self, user, filters): results = { - 'experiments': [], - 'dataformats': {}, - 'common_analyzers': [], - 'common_protocols': [], + "experiments": [], + "dataformats": {}, + "common_analyzers": [], + "common_protocols": [], } if len(filters) == 0: return results - # Use the experiment filters - experiments = Experiment.objects.for_user(user, True).filter(status=Experiment.DONE) + experiments = Experiment.objects.for_user(user, True).filter( + status=Experiment.DONE + ) for filter_entry in filters: experiments = apply_filter(experiments, filter_entry) @@ -375,7 +447,6 @@ class SearchView(APIView): if experiments.count() == 0: return results - # Retrieve informations about each experiment and determine if there is at least # one common analyzer common_protocols = None @@ -383,77 +454,95 @@ class SearchView(APIView): for experiment in experiments.iterator(): experiment_entry = { - 'name': experiment.fullname(), - 'toolchain': experiment.toolchain.fullname(), - 'description': experiment.short_description, - 'public': (experiment.sharing == Shareable.PUBLIC), - 'attestation_number': None, - 'attestation_locked': False, - 'end_date': experiment.end_date, - 'protocols': list(set(map(lambda x: x.protocol.fullname(), experiment.referenced_datasets.iterator()))), - 'analyzers': [], + "name": experiment.fullname(), + "toolchain": experiment.toolchain.fullname(), + "description": experiment.short_description, + "public": (experiment.sharing == Shareable.PUBLIC), + "attestation_number": None, + "attestation_locked": False, + "end_date": experiment.end_date, + "protocols": list( + set( + map( + lambda x: x.protocol.fullname(), + experiment.referenced_datasets.iterator(), + ) + ) + ), + "analyzers": [], } if experiment.has_attestation(): - experiment_entry['attestation_number'] = experiment.attestation.number - experiment_entry['attestation_locked'] = experiment.attestation.locked + experiment_entry["attestation_number"] = experiment.attestation.number + experiment_entry["attestation_locked"] = experiment.attestation.locked experiment_analyzers = [] for analyzer_block in experiment.blocks.filter(analyzer=True).iterator(): analyzer_entry = { - 'name': analyzer_block.algorithm.fullname(), - 'block': analyzer_block.name, - 'results': {}, + "name": analyzer_block.algorithm.fullname(), + "block": analyzer_block.name, + "results": {}, } - experiment_entry['analyzers'].append(analyzer_entry) - experiment_analyzers.append(analyzer_entry['name']) + experiment_entry["analyzers"].append(analyzer_entry) + experiment_analyzers.append(analyzer_entry["name"]) - if analyzer_entry['name'] not in results['dataformats']: - results['dataformats'][analyzer_entry['name']] = json.loads(analyzer_block.algorithm.result_dataformat) + if analyzer_entry["name"] not in results["dataformats"]: + results["dataformats"][analyzer_entry["name"]] = json.loads( + analyzer_block.algorithm.result_dataformat + ) if common_analyzers is None: common_analyzers = experiment_analyzers elif len(common_analyzers) > 0: - common_analyzers = filter(lambda x: x in experiment_analyzers, common_analyzers) + common_analyzers = filter( + lambda x: x in experiment_analyzers, common_analyzers + ) if common_protocols is None: - common_protocols = experiment_entry['protocols'] + common_protocols = experiment_entry["protocols"] elif len(common_protocols) > 0: - common_protocols = filter(lambda x: x in experiment_entry['protocols'], common_protocols) + common_protocols = filter( + lambda x: x in experiment_entry["protocols"], common_protocols + ) - results['experiments'].append(experiment_entry) - - results['common_analyzers'] = common_analyzers - results['common_protocols'] = common_protocols + results["experiments"].append(experiment_entry) + results["common_analyzers"] = common_analyzers + results["common_protocols"] = common_protocols # No common analyzer found, don't retrieve any result if len(common_analyzers) == 0: - results['dataformats'] = {} + results["dataformats"] = {} return results - # Retrieve the results of each experiment for index, experiment in enumerate(experiments.iterator()): for analyzer_block in experiment.blocks.filter(analyzer=True).iterator(): - analyzer_entry = filter(lambda x: x['block'] == analyzer_block.name, - results['experiments'][index]['analyzers'])[0] + analyzer_entry = filter( + lambda x: x["block"] == analyzer_block.name, + results["experiments"][index]["analyzers"], + )[0] for analyzer_result in analyzer_block.results.iterator(): - analyzer_entry['results'][analyzer_result.name] = { - 'type': analyzer_result.type, - 'primary': analyzer_result.primary, - 'value': analyzer_result.value() + analyzer_entry["results"][analyzer_result.name] = { + "type": analyzer_result.type, + "primary": analyzer_result.primary, + "value": analyzer_result.value(), } return results -#------------------------------------------------ +# ------------------------------------------------ -class SearchSaveView(CommonContextMixin, SerializerFieldsMixin, generics.CreateAPIView, generics.UpdateAPIView): +class SearchSaveView( + CommonContextMixin, + SerializerFieldsMixin, + generics.CreateAPIView, + generics.UpdateAPIView, +): """ This endpoint allows to save and update a search query @@ -474,12 +563,12 @@ class SearchSaveView(CommonContextMixin, SerializerFieldsMixin, generics.CreateA fields_to_return = self.get_serializer_fields(request) # Retrieve the description in HTML format - if 'html_description' in fields_to_return: + if "html_description" in fields_to_return: description = search.description if len(description) > 0: - result['html_description'] = ensure_html(description) + result["html_description"] = ensure_html(description) else: - result['html_description'] = '' + result["html_description"] = "" return result def post(self, request): @@ -487,73 +576,91 @@ class SearchSaveView(CommonContextMixin, SerializerFieldsMixin, generics.CreateA serializer.is_valid(raise_exception=True) search = serializer.save() result = self.build_results(request, search) - result['fullname'] = search.fullname() - result['url'] = search.get_absolute_url() + result["fullname"] = search.fullname() + result["url"] = search.get_absolute_url() return Response(result, status=status.HTTP_201_CREATED) def put(self, request, author_name, name): search = get_object_or_404(Search, author__username=author_name, name=name) - serializer = self.get_serializer(instance=search, data=request.data, partial=True) + serializer = self.get_serializer( + instance=search, data=request.data, partial=True + ) serializer.is_valid(raise_exception=True) serializer.save() result = self.build_results(request, search) return Response(result) -#------------------------------------------------ +# ------------------------------------------------ class ListSearchView(CommonContextMixin, generics.ListAPIView): """ Lists all available search from a user """ + permission_classes = [permissions.AllowAny] serializer_class = SearchSerializer def get_queryset(self): - author_name = self.kwargs['author_name'] - return Search.objects.for_user(self.request.user, True).select_related().filter(author__username=author_name) + author_name = self.kwargs["author_name"] + return ( + Search.objects.for_user(self.request.user, True) + .select_related() + .filter(author__username=author_name) + ) -#---------------------------------------------------------- +# ---------------------------------------------------------- -class RetrieveDestroySearchAPIView(CommonContextMixin, SerializerFieldsMixin, IsAuthorOrReadOnlyMixin, generics.RetrieveDestroyAPIView): +class RetrieveDestroySearchAPIView( + CommonContextMixin, + SerializerFieldsMixin, + IsAuthorOrReadOnlyMixin, + generics.RetrieveDestroyAPIView, +): """ Delete the given search """ + model = Search serializer_class = SearchSerializer - def get_object(self): - author_name = self.kwargs.get('author_name') - name = self.kwargs.get('object_name') + author_name = self.kwargs.get("author_name") + name = self.kwargs.get("object_name") user = self.request.user - return get_object_or_404(self.model.objects.for_user(user, True), - author__username=author_name, - name=name) + return get_object_or_404( + self.model.objects.for_user(user, True), + author__username=author_name, + name=name, + ) def get(self, request, *args, **kwargs): search = self.get_object() # Process the query string allow_sharing = request.user == search.author - fields_to_return = self.get_serializer_fields(request, allow_sharing=allow_sharing) + fields_to_return = self.get_serializer_fields( + request, allow_sharing=allow_sharing + ) serializer = self.get_serializer(search, fields=fields_to_return) return Response(serializer.data) -#------------------------------------------------ + +# ------------------------------------------------ class ShareSearchView(ShareView): """ Share the given search with other users/teams """ + model = Search permission_classes = [permissions.AllowAny] def get_queryset(self): - self.kwargs['version'] = 1 + self.kwargs["version"] = 1 return super(ShareSearchView, self).get_queryset()