<commit_before>from functools import partial

from django.conf import settings
from elasticsearch_dsl.field import Keyword
from elasticsearch_dsl.field import Object as DSLObject
from elasticsearch_dsl.field import Text

from kitsune.search.v2.es7_utils import es_analyzer_for_locale

SUPPORTED_LANGUAGES = list(settings.SUMO_LANGUAGES)
# this is a test locale - no need to add it to ES
SUPPORTED_LANGUAGES.remove("xx")


def _get_fields(field, locales, **params):
    """Construct the sub-fields of locale aware multi-field"""
    data = {}

    for locale in locales:
        if field is Text:
            analyzer = es_analyzer_for_locale(locale)
            search_analyzer = es_analyzer_for_locale(locale, search_analyzer=True)
            field_obj = field(analyzer=analyzer, search_analyzer=search_analyzer, **params)
        else:
            field_obj = field(**params)
        data[locale] = field_obj

    return data


def construct_locale_field(field, locales, **params):
    """Construct a locale aware object."""
    inner_fields = _get_fields(locales=locales, field=field, **params)
    return DSLObject(properties=inner_fields)


SumoTextField = partial(construct_locale_field, field=Text)
SumoKeywordField = partial(construct_locale_field, field=Keyword)
# This is an object in the form of
# {'en-US': Text(analyzer_for_the_specific_locale)}
SumoLocaleAwareTextField = partial(SumoTextField, locales=SUPPORTED_LANGUAGES)
SumoLocaleAwareKeywordField = partial(SumoKeywordField, locales=SUPPORTED_LANGUAGES)<commit_msg>don't run quoted strings through synonym graph analyzer (#4746)<commit_after>from functools import partial

from django.conf import settings
from elasticsearch_dsl.field import Keyword
from elasticsearch_dsl.field import Object as DSLObject
from elasticsearch_dsl.field import Text

from kitsune.search.v2.es7_utils import es_analyzer_for_locale

SUPPORTED_LANGUAGES = list(settings.SUMO_LANGUAGES)
# this is a test locale - no need to add it to ES
SUPPORTED_LANGUAGES.remove("xx")


def _get_fields(field, locales, **params):
    """Construct the sub-fields of locale aware multi-field"""
    data = {}

    for locale in locales:
        if field is Text:
            analyzer = es_analyzer_for_locale(locale)
            search_analyzer = es_analyzer_for_locale(locale, search_analyzer=True)
            field_obj = field(
                analyzer=analyzer,
                search_analyzer=search_analyzer,
                search_quote_analyzer=analyzer,
                **params,
            )
        else:
            field_obj = field(**params)
        data[locale] = field_obj

    return data


def construct_locale_field(field, locales, **params):
    """Construct a locale aware object."""
    inner_fields = _get_fields(locales=locales, field=field, **params)
    return DSLObject(properties=inner_fields)


SumoTextField = partial(construct_locale_field, field=Text)
SumoKeywordField = partial(construct_locale_field, field=Keyword)
# This is an object in the form of
# {'en-US': Text(analyzer_for_the_specific_locale)}
SumoLocaleAwareTextField = partial(SumoTextField, locales=SUPPORTED_LANGUAGES)
SumoLocaleAwareKeywordField = partial(SumoKeywordField, locales=SUPPORTED_LANGUAGES)
{'fraction_non_alphanumeric': 0.0783039306716187, 'fraction_numerical': 0.0024760136180748994, 'mean_word_length': 4.404682274247492, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4909409', 'n_tokens_mistral': 982, 'n_tokens_neox': 950, 'n_words': 237}