<commit_before>from operator import attrgetter

from django.conf import settings

import elasticutils
import pyes.exceptions as pyes

from .models import Addon
from compat.models import AppCompat


def extract(addon):
    """Extract indexable attributes from an add-on."""
    attrs = ('id', 'name', 'created', 'last_updated', 'weekly_downloads',
             'bayesian_rating', 'average_daily_users', 'status', 'type',
             'is_disabled')
    d = dict(zip(attrs, attrgetter(*attrs)(addon)))
    # Coerce the Translation into a string.
    d['name'] = unicode(d['name']).lower()
    d['app'] = [a.id for a in addon.compatible_apps]
    # This is an extra query, not good for perf.
    d['category'] = getattr(addon, 'category_ids', [])
    return d


def setup_mapping():
    """Set up the addons index mapping."""
    # Mapping describes how elasticsearch handles a document during indexing.
    # Most fields are detected and mapped automatically.
    m = {
        # Turn off analysis on name so we can sort by it.
        'name': {'index': 'not_analyzed', 'type': 'string'},
    }
    es = elasticutils.get_es()
    try:
        es.create_index_if_missing(settings.ES_INDEX)
    except pyes.ElasticSearchException:
        pass
    try:
        es.put_mapping(Addon._meta.app_label, {'properties': m},
                       settings.ES_INDEX)
    except pyes.ElasticSearchException:
        pass
    try:
        es.put_mapping(AppCompat._meta.app_label, {'properties': m},
                       settings.ES_INDEX)
    except pyes.ElasticSearchException:<commit_msg>make addon.name a multifield with one side having analyzed text<commit_after>from operator import attrgetter

from django.conf import settings

import elasticutils
import pyes.exceptions as pyes

from .models import Addon
from compat.models import AppCompat


def extract(addon):
    """Extract indexable attributes from an add-on."""
    attrs = ('id', 'name', 'created', 'last_updated', 'weekly_downloads',
             'bayesian_rating', 'average_daily_users', 'status', 'type',
             'is_disabled')
    d = dict(zip(attrs, attrgetter(*attrs)(addon)))
    # Coerce the Translation into a string.
    d['name'] = unicode(d['name']).lower()
    d['app'] = [a.id for a in addon.compatible_apps]
    # This is an extra query, not good for perf.
    d['category'] = getattr(addon, 'category_ids', [])
    return d


def setup_mapping():
    """Set up the addons index mapping."""
    # Mapping describes how elasticsearch handles a document during indexing.
    # Most fields are detected and mapped automatically.
    m = {
        # Turn off analysis on name so we can sort by it.
        'name': {
            'type': 'multi_field',
            'fields': {
                'name': {'type': 'string', 'index': 'not_analyzed'},
                'fulltext': {'type': 'string', 'index': 'analyzed'},
            },
        },
    }
    es = elasticutils.get_es()
    try:
        es.create_index_if_missing(settings.ES_INDEX)
    except pyes.ElasticSearchException:
        pass
    try:
        es.put_mapping(Addon._meta.app_label, {'properties': m},
                       settings.ES_INDEX)
    except pyes.ElasticSearchException:
        pass
    try:
        es.put_mapping(AppCompat._meta.app_label, {'properties': m},
                       settings.ES_INDEX)
    except pyes.ElasticSearchException:
{'fraction_non_alphanumeric': 0.11740412979351032, 'fraction_numerical': 0.0, 'mean_word_length': 2.7930648769574944, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2318660', 'n_tokens_mistral': 1006, 'n_tokens_neox': 980, 'n_words': 287}