<commit_before>import logging
from operator import attrgetter

from django.conf import settings

import elasticutils
import pyes.exceptions as pyes

import amo
from .models import Addon
from bandwagon.models import Collection
from compat.models import AppCompat
from users.models import UserProfile


log = logging.getLogger('z.es')


def extract(addon):
    """Extract indexable attributes from an add-on."""
    attrs = ('id', 'created', 'last_updated', 'weekly_downloads',
             'bayesian_rating', 'average_daily_users', 'status', 'type',
             'hotness', 'is_disabled', 'premium_type')
    d = dict(zip(attrs, attrgetter(*attrs)(addon)))
    # Coerce the Translation into a string.
    d['name_sort'] = unicode(addon.name).lower()
    translations = addon.translations
    d['name'] = list(set(string for _, string in translations[addon.name_id]))
    d['description'] = list(set(string for _, string
                                in translations[addon.description_id]))
    d['summary'] = list(set(string for _, string
                            in translations[addon.summary_id]))
    # This is an extra query, not good for perf.
    d['category'] = getattr(addon, 'category_ids', [])
    d['tags'] = getattr(addon, 'tag_list', [])
    if addon.current_version:
        d['platforms'] = [p.id for p in
                          addon.current_version.supported_platforms]
    d['appversion'] = dict((app.id, {'min': appver.min.version_int,
                                     'max': appver.max.version_int})
                           for app, appver in addon.compatible_apps.items()
                           if appver)
    d['app'] = [app.id for app in addon.compatible_apps.keys()]
    # Boost by the number of users on a logarithmic scale. The maximum boost
    # (11,000,000 users for adblock) is about 5x.
    d['_boost'] = addon.average_daily_users ** .2
    # Double the boost if the add-on is public.
    if addon.status == amo.STATUS_PUBLIC:
        d['_boost'] = max(d['_boost'], 1) * 4
    return d


def setup_mapping():
    """Set up the addons index mapping."""
    # Mapping describes how elasticsearch handles a document during indexing.
    # Most fields are detected and mapped automatically.
    appver = {'dynamic': False, 'properties': {'max': {'type': 'long'},
                                               'min': {'type': 'long'}}}
    mapping = {
        # Optional boosting during indexing.
        '_boost': {'name': '_boost', 'null_value': 1.0},
        'properties': {
            # Turn off analysis on name so we can sort by it.
            'name_sort': {'type': 'string', 'index': 'not_analyzed'},
            # Adding word-delimiter to split on camelcase and punctuation.
            'name': {'type': 'string',
                     'analyzer': 'standardPlusWordDelimiter'},
            'tags': {'type': 'string',
                     'index': 'not_analyzed',
                     'index_name': 'tag'},
            'platforms': {'type': 'integer', 'index_name': 'platform'},
            'appversion': {'properties': dict((app.id, appver)
                                              for app in amo.APP_USAGE)},
        },
    }
    es = elasticutils.get_es()
    indexes = settings.ES_INDEXES
    # Adjust the mapping for all models at once because fields are shared
    # across all doc types in an index. If we forget to adjust one of them
    # we'll get burned later on.
    for model in Addon, AppCompat, Collection, UserProfile:
        index = model._get_index()
        try:
            es.create_index_if_missing(index)
        except pyes.ElasticSearchException:
            pass
        try:
            es.put_mapping(model._meta.db_table, mapping, index)
        except pyes.ElasticSearchException, e:
            log.error(e)
<commit_msg>fake app compat for search tools and personas (bug 689848)<commit_after>import logging
from operator import attrgetter

from django.conf import settings

import elasticutils
import pyes.exceptions as pyes

import amo
from .models import Addon
from bandwagon.models import Collection
from compat.models import AppCompat
from users.models import UserProfile


log = logging.getLogger('z.es')


def extract(addon):
    """Extract indexable attributes from an add-on."""
    attrs = ('id', 'created', 'last_updated', 'weekly_downloads',
             'bayesian_rating', 'average_daily_users', 'status', 'type',
             'hotness', 'is_disabled', 'premium_type')
    d = dict(zip(attrs, attrgetter(*attrs)(addon)))
    # Coerce the Translation into a string.
    d['name_sort'] = unicode(addon.name).lower()
    translations = addon.translations
    d['name'] = list(set(string for _, string in translations[addon.name_id]))
    d['description'] = list(set(string for _, string
                                in translations[addon.description_id]))
    d['summary'] = list(set(string for _, string
                            in translations[addon.summary_id]))
    # This is an extra query, not good for perf.
    d['category'] = getattr(addon, 'category_ids', [])
    d['tags'] = getattr(addon, 'tag_list', [])
    if addon.current_version:
        d['platforms'] = [p.id for p in
                          addon.current_version.supported_platforms]
    d['appversion'] = {}
    for app, appver in addon.compatible_apps.items():
        if appver:
            min_, max_ = appver.min.version_int, appver.max.version_int
        else:
            # Fake wide compatibility for search tools and personas.
            min_, max_ = 0, 9999
        d['appversion'][app.id] = dict(min=min_, max=max_)
    d['app'] = [app.id for app in addon.compatible_apps.keys()]
    # Boost by the number of users on a logarithmic scale. The maximum boost
    # (11,000,000 users for adblock) is about 5x.
    d['_boost'] = addon.average_daily_users ** .2
    # Double the boost if the add-on is public.
    if addon.status == amo.STATUS_PUBLIC:
        d['_boost'] = max(d['_boost'], 1) * 4
    return d


def setup_mapping():
    """Set up the addons index mapping."""
    # Mapping describes how elasticsearch handles a document during indexing.
    # Most fields are detected and mapped automatically.
    appver = {'dynamic': False, 'properties': {'max': {'type': 'long'},
                                               'min': {'type': 'long'}}}
    mapping = {
        # Optional boosting during indexing.
        '_boost': {'name': '_boost', 'null_value': 1.0},
        'properties': {
            # Turn off analysis on name so we can sort by it.
            'name_sort': {'type': 'string', 'index': 'not_analyzed'},
            # Adding word-delimiter to split on camelcase and punctuation.
            'name': {'type': 'string',
                     'analyzer': 'standardPlusWordDelimiter'},
            'tags': {'type': 'string',
                     'index': 'not_analyzed',
                     'index_name': 'tag'},
            'platforms': {'type': 'integer', 'index_name': 'platform'},
            'appversion': {'properties': dict((app.id, appver)
                                              for app in amo.APP_USAGE)},
        },
    }
    es = elasticutils.get_es()
    indexes = settings.ES_INDEXES
    # Adjust the mapping for all models at once because fields are shared
    # across all doc types in an index. If we forget to adjust one of them
    # we'll get burned later on.
    for model in Addon, AppCompat, Collection, UserProfile:
        index = model._get_index()
        try:
            es.create_index_if_missing(index)
        except pyes.ElasticSearchException:
            pass
        try:
            es.put_mapping(model._meta.db_table, mapping, index)
        except pyes.ElasticSearchException, e:
            log.error(e)

{'fraction_non_alphanumeric': 0.11921639386518881, 'fraction_numerical': 0.005026420930532285, 'mean_word_length': 2.373913043478261, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4514028', 'n_tokens_mistral': 2251, 'n_tokens_neox': 2165, 'n_words': 666}