<commit_before>from jsonmapping import SchemaVisitor


BASE_MAPPING = {
    "_id": {"path": "id"},
    "_all": {"enabled": True},
    # "_default_": {"type": "string", "store": True, "index": "not_analyzed"},
    "properties": {
        "id": {"type": "string", "index": "not_analyzed"},
        "schema": {"type": "string", "index": "not_analyzed"},
        "indexed_at": {"type": "date", "index": "not_analyzed"},
        "raw": {
            "type": "object",
            "properties": {}
        },
        "source": {"type": "string", "index": "not_analyzed"}
    }
}


def generate_schema_mapping(visitor, path):
    """ Try and recursively iterate a JSON schema and to generate an ES mapping
    that encasulates it. """
    if visitor.is_object:
        mapping = {'type': 'nested', 'properties': {}}
        if not visitor.parent:
            mapping['type'] = 'object'
        if visitor.path in path:
            return mapping
        sub_path = path.union([visitor.path])
        for prop in visitor.properties:
            prop_mapping = generate_schema_mapping(prop, sub_path)
            mapping['properties'][prop.name] = prop_mapping
        return mapping
    elif visitor.is_array:
        for vis in visitor.items:
            return generate_schema_mapping(vis, path)
    else:
        type_name = 'string'
        if 'number' in visitor.types:
            type_name = 'float'
        if 'integer' in visitor.types:
            type_name = 'integer'
        if 'boolean' in visitor.types:
            type_name = 'boolean'
        return {'type': type_name, 'index': 'not_analyzed'}


def generate_mapping(mapping, index, doc_type, record, resolver):
    """ Generate a mapping. """
    mapping = mapping.get(index, {}).get('mappings', {})
    mapping = mapping.get(doc_type, BASE_MAPPING)<commit_msg>make longs for numeric fields. <commit_after>from jsonmapping import SchemaVisitor


BASE_MAPPING = {
    "_id": {"path": "id"},
    "_all": {"enabled": True},
    # "_default_": {"type": "string", "store": True, "index": "not_analyzed"},
    "properties": {
        "id": {"type": "string", "index": "not_analyzed"},
        "schema": {"type": "string", "index": "not_analyzed"},
        "indexed_at": {"type": "date", "index": "not_analyzed"},
        "raw": {
            "type": "object",
            "properties": {}
        },
        "source": {"type": "string", "index": "not_analyzed"}
    }
}


def generate_schema_mapping(visitor, path):
    """ Try and recursively iterate a JSON schema and to generate an ES mapping
    that encasulates it. """
    if visitor.is_object:
        mapping = {'type': 'nested', 'properties': {}}
        if not visitor.parent:
            mapping['type'] = 'object'
        if visitor.path in path:
            return mapping
        sub_path = path.union([visitor.path])
        for prop in visitor.properties:
            prop_mapping = generate_schema_mapping(prop, sub_path)
            mapping['properties'][prop.name] = prop_mapping
        return mapping
    elif visitor.is_array:
        for vis in visitor.items:
            return generate_schema_mapping(vis, path)
    else:
        type_name = 'string'
        if 'number' in visitor.types:
            type_name = 'float'
        if 'integer' in visitor.types:
            type_name = 'long'
        if 'boolean' in visitor.types:
            type_name = 'boolean'
        return {'type': type_name, 'index': 'not_analyzed'}


def generate_mapping(mapping, index, doc_type, record, resolver):
    """ Generate a mapping. """
    mapping = mapping.get(index, {}).get('mappings', {})
    mapping = mapping.get(doc_type, BASE_MAPPING)
{'fraction_non_alphanumeric': 0.14941272876263317, 'fraction_numerical': 0.0, 'mean_word_length': 2.5937193326790973, 'pattern_counts': {'":': 48, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6941549', 'n_tokens_mistral': 1078, 'n_tokens_neox': 1067, 'n_words': 288}