<commit_before>boolean_notanalyzed = {'type': 'boolean', 'index': 'not_analyzed'}
integer_notanalyzed = {'type': 'integer', 'index': 'not_analyzed'}
keyword_notanalyzed = {'type': 'keyword', 'index': 'not_analyzed'}

DOC_TYPE = 'failure-line'
INDEX_NAME = 'failure-lines'

INDEX_SETTINGS = {
    'failure-lines': {
        'mappings': {
            'failure-line': {
                'properties': {
                    'job_guid': keyword_notanalyzed,
                    'test': keyword_notanalyzed,
                    'subtest': keyword_notanalyzed,
                    'status': keyword_notanalyzed,
                    'expected': keyword_notanalyzed,
                    'best_classification': integer_notanalyzed,
                    'best_is_verified': boolean_notanalyzed,
                    'message': {
                        'type': 'text',
                        'analyzer': 'message_analyzer',
                        'search_analyzer': 'message_analyzer',
                    },
                },
            },
        },
        'settings': {
            'number_of_shards': 1,
            'analysis': {
                'analyzer': {
                    'message_analyzer': {
                        'type': 'custom',
                        'tokenizer': 'message_tokenizer',
                        'filters': [],
                    },
                },
                'tokenizer': {
                    'message_tokenizer': {
                        'type': 'pattern',
                        'pattern': r'0x[0-9a-fA-F]+|[\W0-9]+?',<commit_msg>Bug 1427598 - Remove legacy index options<commit_after>boolean = {'type': 'boolean'}
integer = {'type': 'integer'}
keyword = {'type': 'keyword'}

DOC_TYPE = 'failure-line'
INDEX_NAME = 'failure-lines'

INDEX_SETTINGS = {
    'failure-lines': {
        'mappings': {
            'failure-line': {
                'properties': {
                    'job_guid': keyword,
                    'test': keyword,
                    'subtest': keyword,
                    'status': keyword,
                    'expected': keyword,
                    'best_classification': integer,
                    'best_is_verified': boolean,
                    'message': {
                        'type': 'text',
                        'analyzer': 'message_analyzer',
                        'search_analyzer': 'message_analyzer',
                    },
                },
            },
        },
        'settings': {
            'number_of_shards': 1,
            'analysis': {
                'analyzer': {
                    'message_analyzer': {
                        'type': 'custom',
                        'tokenizer': 'message_tokenizer',
                        'filters': [],
                    },
                },
                'tokenizer': {
                    'message_tokenizer': {
                        'type': 'pattern',
                        'pattern': r'0x[0-9a-fA-F]+|[\W0-9]+?',
{'fraction_non_alphanumeric': 0.13286242852337707, 'fraction_numerical': 0.006390850992263707, 'mean_word_length': 1.1676384839650147, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4660054', 'n_tokens_mistral': 794, 'n_tokens_neox': 733, 'n_words': 122}