<commit_before>"""\
Example.

To load the initial data:

    %(prog)s production.ini

"""
from pyramid.paster import get_app
from elasticsearch import RequestError
from contentbase import (
    COLLECTIONS,
    TYPES,
)
from contentbase.util import ensurelist
from .interfaces import ELASTIC_SEARCH
import collections
import json
import logging

EPILOG = __doc__

log = logging.getLogger(__name__)

# An index to store non-content metadata
META_MAPPING = {
    'dynamic_templates': [
        {
            'store_generic': {
                'match': '*',
                'mapping': {
                    'index': 'no',
                    'store': 'yes',
                },
            },
        },
    ],
}


def sorted_pairs_hook(pairs):
    return collections.OrderedDict(sorted(pairs))


def sorted_dict(d):
    return json.loads(json.dumps(d), object_pairs_hook=sorted_pairs_hook)


def schema_mapping(name, schema):
    if 'linkFrom' in schema:
        type_ = 'string'
    else:
        type_ = schema['type']

    # Elasticsearch handles multiple values for a field
    if type_ == 'array' and schema['items']:
        return schema_mapping(name, schema['items'])

    if type_ == 'object':
        properties = {}
        for k, v in schema.get('properties', {}).items():
            mapping = schema_mapping(k, v)
            if mapping is not None:
                properties[k] = mapping
        return {
            'type': 'object',
            'include_in_all': False,
            'properties': properties,
        }

    if type_ == ["number", "string"]:
        return {
            'type': 'string',
            'copy_to': [],
            'index': 'not_analyzed',
            'fields': {
                'value': {
                    'type': 'float',
                    'copy_to': '',
                    'ignore_malformed': True,
                    'copy_to': []
                },
                'raw': {
                    'type': 'string',
                    'index': 'not_analyzed'
                }
            }
        }

    if type_ in ['string', 'boolean']:
        return {
            'type': 'string',
            'store': True,
            'fields': {
                'raw': {
                    'type': 'string',
                    'index': 'not_analyzed'
                }
            }
        }

    if type_ == 'number':
        return {
            'type': 'float',
            'store': True,
            'fields': {
                'raw': {
                    'type': 'string',
                    'index': 'not_analyzed'
                }
            }
        }

    if type_ == 'integer':
        return {
            'type': 'long',
            'store': True,
            'fields': {
                'raw': {
                    'type': 'string',
                    'index': 'not_analyzed'
                }
            }
        }


def index_settings():
    return {
        'index': {<commit_msg>Changed Elasticsearch settings to increase efficiency<commit_after>"""\
Example.

To load the initial data:

    %(prog)s production.ini

"""
from pyramid.paster import get_app
from elasticsearch import RequestError
from contentbase import (
    COLLECTIONS,
    TYPES,
)
from contentbase.util import ensurelist
from .interfaces import ELASTIC_SEARCH
import collections
import json
import logging

EPILOG = __doc__

log = logging.getLogger(__name__)

# An index to store non-content metadata
META_MAPPING = {
    'dynamic_templates': [
        {
            'store_generic': {
                'match': '*',
                'mapping': {
                    'index': 'no',
                    'store': 'yes',
                },
            },
        },
    ],
}


def sorted_pairs_hook(pairs):
    return collections.OrderedDict(sorted(pairs))


def sorted_dict(d):
    return json.loads(json.dumps(d), object_pairs_hook=sorted_pairs_hook)


def schema_mapping(name, schema):
    if 'linkFrom' in schema:
        type_ = 'string'
    else:
        type_ = schema['type']

    # Elasticsearch handles multiple values for a field
    if type_ == 'array' and schema['items']:
        return schema_mapping(name, schema['items'])

    if type_ == 'object':
        properties = {}
        for k, v in schema.get('properties', {}).items():
            mapping = schema_mapping(k, v)
            if mapping is not None:
                properties[k] = mapping
        return {
            'type': 'object',
            'include_in_all': False,
            'properties': properties,
        }

    if type_ == ["number", "string"]:
        return {
            'type': 'string',
            'copy_to': [],
            'index': 'not_analyzed',
            'fields': {
                'value': {
                    'type': 'float',
                    'copy_to': '',
                    'ignore_malformed': True,
                    'copy_to': []
                },
                'raw': {
                    'type': 'string',
                    'index': 'not_analyzed'
                }
            }
        }

    if type_ in ['string', 'boolean']:
        return {
            'type': 'string',
            'store': True,
            'fields': {
                'raw': {
                    'type': 'string',
                    'index': 'not_analyzed'
                }
            }
        }

    if type_ == 'number':
        return {
            'type': 'float',
            'store': True,
            'fields': {
                'raw': {
                    'type': 'string',
                    'index': 'not_analyzed'
                }
            }
        }

    if type_ == 'integer':
        return {
            'type': 'long',
            'store': True,
            'fields': {
                'raw': {
                    'type': 'string',
                    'index': 'not_analyzed'
                }
            }
        }


def index_settings():
    return {
        'index': {
            'number_of_shards': 1,
            'merge': {
                'policy': {
                    'max_merged_segment': '2gb',
                    'max_merge_at_once': 5
                }
            },
{'fraction_non_alphanumeric': 0.12502029550251664, 'fraction_numerical': 0.0004870920603994155, 'mean_word_length': 1.2797927461139897, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3130656', 'n_tokens_mistral': 1743, 'n_tokens_neox': 1660, 'n_words': 391}