<commit_before>""" These are utility functions used by the OCCRP datamapper to generate a
matching ElasticSearch schema, given a JSON Schema descriptor. """
from jsonmapping.visitor import SchemaVisitor


def generate_schema_mapping(resolver, schema_uri):
    """ Try and recursively iterate a JSON schema and to generate an ES mapping
    that encasulates it. """
    visitor = SchemaVisitor({'$ref': schema_uri}, resolver)
    return _generate_schema_mapping(visitor, set())


def _generate_schema_mapping(visitor, path):
    if visitor.is_object:
        mapping = {
            'type': 'nested',
            '_id': {'path': 'id'},
            'properties': {
                '$schema': {'type': 'string', 'index': 'not_analyzed'},
                'id': {'type': 'string', 'index': 'not_analyzed'}
            }
        }
        if not visitor.parent:
            mapping['type'] = 'object'
        if visitor.path in path:
            return mapping
        sub_path = path.union([visitor.path])
        for prop in visitor.properties:
            prop_mapping = _generate_schema_mapping(prop, sub_path)
            mapping['properties'][prop.name] = prop_mapping
        return mapping
    elif visitor.is_array:
        for vis in visitor.items:
            return _generate_schema_mapping(vis, path)
    else:
        type_name = 'string'
        if 'number' in visitor.types:
            type_name = 'float'
        if 'integer' in visitor.types:
            type_name = 'long'
        if 'boolean' in visitor.types:
            type_name = 'boolean'
        mapping = {'type': type_name, 'index': 'not_analyzed'}
        format_ = visitor.schema.get('format')
        if format_ and format_ in ('date-time', 'datetime', 'date'):
            mapping['type'] = 'date'
            mapping['format'] = 'dateOptionalTime'
        return mapping<commit_msg>Adapt elastic schema generator to new API. <commit_after>""" These are utility functions used by the OCCRP datamapper to generate a
matching ElasticSearch schema, given a JSON Schema descriptor. """
from jsonmapping.visitor import SchemaVisitor


def generate_schema_mapping(resolver, schema_uri):
    """ Try and recursively iterate a JSON schema and to generate an ES mapping
    that encasulates it. """
    visitor = SchemaVisitor({'$ref': schema_uri}, resolver)
    return _generate_schema_mapping(visitor, set())


def _generator_field_mapping(visitor):
    type_name = 'string'
    if 'number' in visitor.types:
        type_name = 'float'
    if 'integer' in visitor.types:
        type_name = 'long'
    if 'boolean' in visitor.types:
        type_name = 'boolean'
    mapping = {'type': type_name, 'index': 'not_analyzed'}
    format_ = visitor.schema.get('format')
    if format_ and format_ in ('date-time', 'datetime', 'date'):
        mapping['type'] = 'date'
        mapping['format'] = 'dateOptionalTime'
    return mapping


def _generate_schema_mapping(visitor, path):
    if visitor.is_object:
        mapping = {
            'type': 'nested',
            '_id': {'path': 'id'},
            'properties': {
                '$schema': {'type': 'string', 'index': 'not_analyzed'},
                'id': {'type': 'string', 'index': 'not_analyzed'}
            }
        }
        if not visitor.parent:
            mapping['type'] = 'object'
        if visitor.path in path:
            return mapping
        sub_path = path.union([visitor.path])
        for prop in visitor.properties:
            prop_mapping = _generate_schema_mapping(prop, sub_path)
            mapping['properties'][prop.name] = prop_mapping
        return mapping
    elif visitor.is_array:
        return _generate_schema_mapping(visitor.items, path)
    else:
        return _generator_field_mapping(visitor)
{'fraction_non_alphanumeric': 0.11403508771929824, 'fraction_numerical': 0.0, 'mean_word_length': 2.5940783190066856, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6681283', 'n_tokens_mistral': 1043, 'n_tokens_neox': 1028, 'n_words': 302}