<commit_before>from corehq.pillows.base import DEFAULT_META
from corehq.pillows.core import DATE_FORMATS_ARR, DATE_FORMATS_STRING
from corehq.util.elastic import es_index
from pillowtop.es_utils import ElasticsearchIndexInfo

SMS_INDEX = es_index("smslogs_708c77f8e5fe00286fa5791e9fa7d45f")
SMS_MAPPING = {
    '_meta': {
       'comment': 'Autogenerated [corehq.apps.sms.models.SMSLog] mapping from ptop_generate_mapping 09/23/2013',
       'created': None
    },
    'date_detection': False,
    'date_formats': DATE_FORMATS_ARR,
    'dynamic': False,
    'properties': {
        'backend_api': {'type': 'string'},
        'backend_id': {'type': 'string'},
        'base_doc': {'type': 'string'},
        'billed': {'type': 'boolean'},
        'couch_recipient': {'type': 'string'},
        'couch_recipient_doc_type': {'type': 'string'},
        'date': {
            'format': DATE_FORMATS_STRING,
            'type': 'date'
        },
        'direction': {'type': 'string'},
        'doc_type': {'index': 'not_analyzed', 'type': 'string'},
        'domain': {
            'fields': {
                'domain': {'index': 'analyzed', 'type': 'string'},
                'exact': {'index': 'not_analyzed', 'type': 'string'}
            },
            'type': 'multi_field'
        },
        'phone_number': {'type': 'string'},
        'reminder_id': {'type': 'string'},
        'text': {'type': 'string'},
        'workflow': {'type': 'string'},
        'xforms_session_couch_id': {'type': 'string'}
    }<commit_msg>include processed in sms index<commit_after>from corehq.pillows.base import DEFAULT_META
from corehq.pillows.core import DATE_FORMATS_ARR, DATE_FORMATS_STRING
from corehq.util.elastic import es_index
from pillowtop.es_utils import ElasticsearchIndexInfo

SMS_INDEX = es_index("smslogs_2017-02-09")
SMS_MAPPING = {
    '_meta': {
       'comment': 'Autogenerated [corehq.apps.sms.models.SMSLog] mapping from ptop_generate_mapping 09/23/2013',
       'created': None
    },
    'date_detection': False,
    'date_formats': DATE_FORMATS_ARR,
    'dynamic': False,
    'properties': {
        'backend_api': {'type': 'string'},
        'backend_id': {'type': 'string'},
        'base_doc': {'type': 'string'},
        'billed': {'type': 'boolean'},
        'couch_recipient': {'type': 'string'},
        'couch_recipient_doc_type': {'type': 'string'},
        'date': {
            'format': DATE_FORMATS_STRING,
            'type': 'date'
        },
        'direction': {'type': 'string'},
        'doc_type': {'index': 'not_analyzed', 'type': 'string'},
        'domain': {
            'fields': {
                'domain': {'index': 'analyzed', 'type': 'string'},
                'exact': {'index': 'not_analyzed', 'type': 'string'}
            },
            'type': 'multi_field'
        },
        'phone_number': {'type': 'string'},
        'processed': {'type': 'boolean'},
        'reminder_id': {'type': 'string'},
        'text': {'type': 'string'},
        'workflow': {'type': 'string'},
        'xforms_session_couch_id': {'type': 'string'}
    }
{'fraction_non_alphanumeric': 0.18037049073773156, 'fraction_numerical': 0.014299642508937277, 'mean_word_length': 2.828358208955224, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3732230', 'n_tokens_mistral': 1047, 'n_tokens_neox': 994, 'n_words': 201}