<commit_before>import itertools

from django.conf import settings

from rest_framework.decorators import api_view
from rest_framework.response import Response

from kitsune.products.models import Product
from kitsune.questions.models import QuestionMappingType
from kitsune.search import es_utils
from kitsune.sumo.api import GenericAPIException
from kitsune.wiki.models import DocumentMappingType


@api_view(['GET', 'POST'])
def suggest(request):
    text = request.body or request.GET.get('q')
    locale = request.GET.get('locale', settings.WIKI_DEFAULT_LANGUAGE)
    product = request.GET.get('product')
    max_questions = request.GET.get('max_questions', 10)
    max_documents = request.GET.get('max_documents', 10)

    errors = {}
    try:
        max_questions = int(max_questions)
    except ValueError:
        errors['max_questions'] = 'This field must be an integer.'
    try:
        max_documents = int(max_documents)
    except ValueError:
        errors['max_documents'] = 'This field must be an integer.'
    if text is None:
        errors['q'] = 'This field is required.'
    if product is not None and not Product.objects.filter(slug=product).exists():
        errors['product'] = 'Could not find product with slug "{0}".'.format(product)
    if errors:
        raise GenericAPIException(400, errors)

    wiki_f = es_utils.F(
        model='wiki_document',
        document_category__in=settings.SEARCH_DEFAULT_CATEGORIES,
        document_locale=locale,
        document_is_archived=False)

    questions_f = es_utils.F(
        model='questions_question',
        question_is_archived=False,
        question_is_locked=False,
        question_has_helpful=True)

    if product is not None:
        wiki_f &= es_utils.F(product=product)
        questions_f &= es_utils.F(product=product)

    mapping_types = [QuestionMappingType, DocumentMappingType]
    query_fields = itertools.chain(*[cls.get_query_fields() for cls in mapping_types])
    query = {}
    for field in query_fields:
        for query_type in ['match', 'match_phrase']:
            key = '{0}__{1}'.format(field, query_type)
            query[key] = text

    # Transform query to be locale aware.
    query = es_utils.es_query_with_analyzer(query, locale)

    searcher = (
        es_utils.AnalyzerS()
        .es(urls=settings.ES_URLS)
        .indexes(es_utils.read_index('default'))
        .doctypes(*[cls.get_mapping_type_name() for cls in mapping_types])
        .filter(wiki_f | questions_f)
        .query(should=True, **query))

    documents = []
    questions = []

    for result in searcher[:(max_documents + max_questions) * 2]:
        if result['model'] == 'wiki_document':
            documents.append({
                'title': result['document_title'],
                'slug': result['document_slug'],
                'summary': result['document_summary'],
            })
        elif result['model'] == 'questions_question':
            questions.append({
                'id': result['id'],
                'title': result['question_title'],
            })
        if len(documents) >= max_documents and len(questions) >= max_questions:
            break

    return Response({
        'questions': questions[:max_questions],
        'documents': documents[:max_documents],
    })
<commit_msg>[Bug 1122539] Refactor search suggestion API to make separate queries.<commit_after>from django.conf import settings

from rest_framework.decorators import api_view
from rest_framework.response import Response

from kitsune.products.models import Product
from kitsune.questions.models import QuestionMappingType
from kitsune.search import es_utils
from kitsune.sumo.api import GenericAPIException
from kitsune.wiki.models import DocumentMappingType


@api_view(['GET', 'POST'])
def suggest(request):
    text = request.body or request.GET.get('q')
    locale = request.GET.get('locale', settings.WIKI_DEFAULT_LANGUAGE)
    product = request.GET.get('product')
    max_questions = request.GET.get('max_questions', 10)
    max_documents = request.GET.get('max_documents', 10)

    errors = {}
    try:
        max_questions = int(max_questions)
    except ValueError:
        errors['max_questions'] = 'This field must be an integer.'
    try:
        max_documents = int(max_documents)
    except ValueError:
        errors['max_documents'] = 'This field must be an integer.'
    if text is None:
        errors['q'] = 'This field is required.'
    if product is not None and not Product.objects.filter(slug=product).exists():
        errors['product'] = 'Could not find product with slug "{0}".'.format(product)
    if errors:
        raise GenericAPIException(400, errors)

    searcher = (
        es_utils.AnalyzerS()
        .es(urls=settings.ES_URLS)
        .indexes(es_utils.read_index('default')))

    return Response({
        'questions': _question_suggestions(searcher, text, locale, product, max_questions),
        'documents': _document_suggestions(searcher, text, locale, product, max_documents),
    })


def _question_suggestions(searcher, text, locale, product, max_results):
    search_filter = es_utils.F(
        model='questions_question',
        question_is_archived=False,
        question_is_locked=False,
        question_has_helpful=True)

    if product is not None:
        search_filter &= es_utils.F(product=product)

    questions = []
    searcher = _query(searcher, QuestionMappingType, search_filter, text, locale)

    for result in searcher[:max_results]:
        questions.append({
            'id': result['id'],
            'title': result['question_title'],
        })

    return questions


def _document_suggestions(searcher, text, locale, product, max_results):
    search_filter = es_utils.F(
        model='wiki_document',
        document_category__in=settings.SEARCH_DEFAULT_CATEGORIES,
        document_locale=locale,
        document_is_archived=False)

    if product is not None:
        search_filter &= es_utils.F(product=product)

    documents = []
    searcher = _query(searcher, DocumentMappingType, search_filter, text, locale)

    for result in searcher[:max_results]:
        documents.append({
            'title': result['document_title'],
            'slug': result['document_slug'],
            'summary': result['document_summary'],
        })

    return documents


def _query(searcher, mapping_type, search_filter, query_text, locale):
    query_fields = mapping_type.get_query_fields()
    query = {}
    for field in query_fields:
        for query_type in ['match', 'match_phrase']:
            key = '{0}__{1}'.format(field, query_type)
            query[key] = query_text

    # Transform query to be locale aware.
    query = es_utils.es_query_with_analyzer(query, locale)

    return (searcher
            .doctypes(mapping_type.get_mapping_type_name())
            .filter(search_filter)
            .query(should=True, **query))

{'fraction_non_alphanumeric': 0.10752998988293105, 'fraction_numerical': 0.004046827576239341, 'mean_word_length': 3.1636582430806257, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4367028', 'n_tokens_mistral': 2055, 'n_tokens_neox': 1979, 'n_words': 461}