<commit_before>"""\
Example.

To load the initial data:

    %(prog)s production.ini

"""
from pyramid.paster import get_app
from pyramid.traversal import find_root
from elasticsearch import RequestError
from ..indexing import ELASTIC_SEARCH
import collections
import json
import logging

EPILOG = __doc__

log = logging.getLogger(__name__)

# An index to store non-content metadata
META_MAPPING = {
    'dynamic_templates': [
        {
            'store_generic': {
                'match': '*',
                'mapping': {
                    'index': 'no',
                    'store': 'yes',
                },
            },
        },
    ],
}


def sorted_pairs_hook(pairs):
    return collections.OrderedDict(sorted(pairs))


def sorted_dict(d):
    return json.loads(json.dumps(d), object_pairs_hook=sorted_pairs_hook)


def schema_mapping(name, schema):
    type_ = schema['type']

    # Elasticsearch handles multiple values for a field
    if type_ == 'array' and schema['items']:
        return schema_mapping(name, schema['items'])

    if type_ == 'object':
        properties = {}
        for k, v in schema['properties'].items():
            mapping = schema_mapping(k, v)
            if mapping is not None:
                properties[k] = mapping
        return {'properties': properties}

    if type_ == ["number", "string"]:
        return {
            'type': 'string',
            'include_in_all': False,
            'copy_to': [],
            'index': 'not_analyzed',
            'fields': {
                'value': {
                    'type': 'float',
                    'copy_to': '',
                    'ignore_malformed': True,
                    'include_in_all': False,
                    'copy_to': []
                }
            }
        }

    if type_ == 'string':
        return {
            'type': 'string',
            'include_in_all': False,
            'copy_to': [],
            'index': 'not_analyzed',
        }

    if type_ == 'number':
        return {
            'type': 'float',
            'copy_to': [],
            'include_in_all': False
        }

    if type_ in ('boolean', 'integer'):
        return {
            'type': type_,
            'copy_to': [],
            'include_in_all': False
        }


def index_settings():
    return {
        'index': {
            'analysis': {
                'filter': {
                    'substring': {
                        'type': 'nGram',
                        'min_gram': 1,
                        'max_gram': 33
                    }
                },
                'analyzer': {
                    'encoded_index_analyzer': {
                        'type': 'custom',
                        'tokenizer': 'standard',
                        'filter': [
                            'standard',
                            'lowercase',
                            'asciifolding',
                            'substring'
                        ]
                    },
                    'encoded_search_analyzer': {
                        'type': 'custom',
                        'tokenizer': 'standard',
                        'filter': [
                            'standard',
                            'lowercase',
                            'asciifolding'
                        ]
                    }
                }
            }<commit_msg>strip html tags for encoded_all_standard and encoded_all_ngram<commit_after>"""\
Example.

To load the initial data:

    %(prog)s production.ini

"""
from pyramid.paster import get_app
from pyramid.traversal import find_root
from elasticsearch import RequestError
from ..indexing import ELASTIC_SEARCH
import collections
import json
import logging

EPILOG = __doc__

log = logging.getLogger(__name__)

# An index to store non-content metadata
META_MAPPING = {
    'dynamic_templates': [
        {
            'store_generic': {
                'match': '*',
                'mapping': {
                    'index': 'no',
                    'store': 'yes',
                },
            },
        },
    ],
}


def sorted_pairs_hook(pairs):
    return collections.OrderedDict(sorted(pairs))


def sorted_dict(d):
    return json.loads(json.dumps(d), object_pairs_hook=sorted_pairs_hook)


def schema_mapping(name, schema):
    type_ = schema['type']

    # Elasticsearch handles multiple values for a field
    if type_ == 'array' and schema['items']:
        return schema_mapping(name, schema['items'])

    if type_ == 'object':
        properties = {}
        for k, v in schema['properties'].items():
            mapping = schema_mapping(k, v)
            if mapping is not None:
                properties[k] = mapping
        return {'properties': properties}

    if type_ == ["number", "string"]:
        return {
            'type': 'string',
            'include_in_all': False,
            'copy_to': [],
            'index': 'not_analyzed',
            'fields': {
                'value': {
                    'type': 'float',
                    'copy_to': '',
                    'ignore_malformed': True,
                    'include_in_all': False,
                    'copy_to': []
                }
            }
        }

    if type_ == 'string':
        return {
            'type': 'string',
            'include_in_all': False,
            'copy_to': [],
            'index': 'not_analyzed',
        }

    if type_ == 'number':
        return {
            'type': 'float',
            'copy_to': [],
            'include_in_all': False
        }

    if type_ in ('boolean', 'integer'):
        return {
            'type': type_,
            'copy_to': [],
            'include_in_all': False
        }


def index_settings():
    return {
        'index': {
            'analysis': {
                'filter': {
                    'substring': {
                        'type': 'nGram',
                        'min_gram': 1,
                        'max_gram': 33
                    }
                },
                'analyzer': {
                    'default': {
                        'type': 'custom',
                        'tokenizer': 'standard',
                        'char_filter': 'html_strip',
                        'filter': [
                            'standard',
                            'lowercase',
                        ]
                    },
                    'encoded_index_analyzer': {
                        'type': 'custom',
                        'tokenizer': 'standard',
                        'char_filter': 'html_strip',
                        'filter': [
                            'standard',
                            'lowercase',
                            'asciifolding',
                            'substring'
                        ]
                    },
                    'encoded_search_analyzer': {
                        'type': 'custom',
                        'tokenizer': 'standard',
                        'filter': [
                            'standard',
                            'lowercase',
                            'asciifolding'
                        ]
                    }
                }
            }
{'fraction_non_alphanumeric': 0.11470873114708731, 'fraction_numerical': 0.0008302200083022001, 'mean_word_length': 1.0847995385059128, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7234438', 'n_tokens_mistral': 1890, 'n_tokens_neox': 1793, 'n_words': 392}