<commit_before># -*- coding: utf-8 -*-
import elasticsearch_dsl


class Geocomplete(elasticsearch_dsl.DocType):
    class Meta:
        index = 'geocomplete'
        doc_type = 'geoloc-entry'

    french_elision = elasticsearch_dsl.token_filter(
        'french_elision',
        type='elision',
        articles_case=True,
        articles=[
            'l', 'm', 't', 'qu', 'n', 's',
            'j', 'd', 'c', 'jusqu', 'quoiqu',
            'lorsqu', 'puisqu'
        ]
    )

    geocompletion_ngram_filter = elasticsearch_dsl.token_filter(
        'geocompletion_ngram',
        type='edgeNGram',
        min_gram=1,
        max_gram=50,
        side='front'
    )

    geocompletion_index_analyzer = elasticsearch_dsl.analyzer(
        'geocompletion_index_analyzer',
        type='custom',
        tokenizer='standard',
        filter=[
            'lowercase',
            'asciifolding',
            'word_delimiter',
            french_elision,
            geocompletion_ngram_filter
        ]
    )

    geocompletion_search_analyzer = elasticsearch_dsl.analyzer(
        'geocompletion_search_analyzer',
        type='custom',
        tokenizer='standard',
        filter=[
            'lowercase',
            'asciifolding'
        ]
    )

    name = elasticsearch_dsl.String(
        index='analyzed',
        analyzer=geocompletion_index_analyzer,
        search_analyzer=geocompletion_search_analyzer,
        fields=dict(raw=elasticsearch_dsl.String(index='not_analyzed'))
    )

    postal_code_ngram_filter = elasticsearch_dsl.token_filter(
        'postal_code_ngram',
        type='edgeNGram',
        min_gram=1,
        max_gram=5,
        side='front'
    )

    postal_code_index_analyzer = elasticsearch_dsl.analyzer(
        'postal_code_index_analyzer',
        type='custom',
        tokenizer='standard',
        filter=[
            postal_code_ngram_filter
        ]
    )

    postal_code_search_analyzer = elasticsearch_dsl.analyzer(
        'postal_code_search_analyzer',
        type='custom',
        tokenizer='standard',
        filter=[
            'lowercase',
            'asciifolding'
        ]
    )

    postal_code = elasticsearch_dsl.String(
        index='analyzed',
        analyzer=postal_code_index_analyzer,
        search_analyzer=postal_code_search_analyzer
    )
<commit_msg>Postal code search analyzer simplification<commit_after># -*- coding: utf-8 -*-
import elasticsearch_dsl


class Geocomplete(elasticsearch_dsl.DocType):
    class Meta:
        index = 'geocomplete'
        doc_type = 'geoloc-entry'

    french_elision = elasticsearch_dsl.token_filter(
        'french_elision',
        type='elision',
        articles_case=True,
        articles=[
            'l', 'm', 't', 'qu', 'n', 's',
            'j', 'd', 'c', 'jusqu', 'quoiqu',
            'lorsqu', 'puisqu'
        ]
    )

    geocompletion_ngram_filter = elasticsearch_dsl.token_filter(
        'geocompletion_ngram',
        type='edgeNGram',
        min_gram=1,
        max_gram=50,
        side='front'
    )

    geocompletion_index_analyzer = elasticsearch_dsl.analyzer(
        'geocompletion_index_analyzer',
        type='custom',
        tokenizer='standard',
        filter=[
            'lowercase',
            'asciifolding',
            'word_delimiter',
            french_elision,
            geocompletion_ngram_filter
        ]
    )

    geocompletion_search_analyzer = elasticsearch_dsl.analyzer(
        'geocompletion_search_analyzer',
        type='custom',
        tokenizer='standard',
        filter=[
            'lowercase',
            'asciifolding'
        ]
    )

    name = elasticsearch_dsl.String(
        index='analyzed',
        analyzer=geocompletion_index_analyzer,
        search_analyzer=geocompletion_search_analyzer,
        fields=dict(raw=elasticsearch_dsl.String(index='not_analyzed'))
    )

    postal_code_ngram_filter = elasticsearch_dsl.token_filter(
        'postal_code_ngram',
        type='edgeNGram',
        min_gram=1,
        max_gram=5,
        side='front'
    )

    postal_code_index_analyzer = elasticsearch_dsl.analyzer(
        'postal_code_index_analyzer',
        type='custom',
        tokenizer='standard',
        filter=[
            postal_code_ngram_filter
        ]
    )

    postal_code_search_analyzer = elasticsearch_dsl.analyzer(
        'postal_code_search_analyzer',
        type='custom',
        tokenizer='standard'
    )

    postal_code = elasticsearch_dsl.String(
        index='analyzed',
        analyzer=postal_code_index_analyzer,
        search_analyzer=postal_code_search_analyzer
    )

{'fraction_non_alphanumeric': 0.10162513542795233, 'fraction_numerical': 0.002600216684723727, 'mean_word_length': 2.248416608022519, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5095753', 'n_tokens_mistral': 1468, 'n_tokens_neox': 1451, 'n_words': 183}