<commit_before>package org.elasticsearch.description;

import net.itimothy.rest.description.Model;
import net.itimothy.rest.description.Primitive;
import net.itimothy.rest.description.Property;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.hppc.cursors.ObjectCursor;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import static java.util.Arrays.asList;

public class ModelsCatalog {

    private final DataProvider dataProvider;
    private String indexOrAlias;

    Map<String, List<Model>> indexTypeModelsMap;
    Map<String, List<Model>> indexDocumentModelsMap;

    public ModelsCatalog(DataProvider dataProvider, String indexOrAlias) {
        this.dataProvider = dataProvider;
        this.indexOrAlias = indexOrAlias;
    }

    public Map<String, List<Model>> getIndexTypeModelsMap() {
        if (indexTypeModelsMap == null) {
            indexTypeModelsMap = new HashMap<>();

            GetMappingsResponse getMappingsResponse = dataProvider.getClient().admin().indices().prepareGetMappings().get();

            ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> indexTypeMappings = getMappingsResponse.getMappings();
            for (ObjectCursor<String> indexCursor : indexTypeMappings.keys()) {
                String indexName = indexCursor.value;
                List<Model> typeModels = new ArrayList<>();
                indexTypeModelsMap.put(indexName, typeModels);

                ImmutableOpenMap<String, MappingMetaData> typeMappings = indexTypeMappings.get(indexName);
                for (ObjectCursor<String> typeCursor : typeMappings.keys()) {
                    String typeName = typeCursor.value;

                    Map mappingProperties = null;
                    try {
                        mappingProperties = (Map) typeMappings.get(typeCursor.value).getSourceAsMap().get("properties");
                    } catch (IOException e) {
                        e.printStackTrace();
                    }

                    List<Property> properties = new ArrayList<>();
                    for (Object propertyName : mappingProperties.keySet()) {
                        String mappingType = ((Map) mappingProperties.get(propertyName.toString())).get("type").toString();

                        properties.add(
                            Property.builder()
                                .name(propertyName.toString())
                                .model(mappingTypeToModel(mappingType)).build()
                        );
                    }

                    typeModels.add(
                        Model.builder()
                            .id(getTypeModelId(indexName, typeName))
                            .name(typeName)
                            .properties(properties)
                            .build()
                    );
                }
            }
        }

        return indexTypeModelsMap;
    }

    public Map<String, List<Model>> getIndexDocumentModelsMap() {
        if (indexDocumentModelsMap == null) {
            indexDocumentModelsMap = getIndexTypeModelsMap().entrySet().stream()
                .collect(
                    Collectors.toMap(
                        e -> e.getKey(),
                        e -> {
                            String indexName = e.getKey();
                            List<Model> typeModels = e.getValue();

                            return typeModels.stream()<commit_msg>More defense resolving of mapping's field type and fallback to "string"<commit_after>package org.elasticsearch.description;

import net.itimothy.rest.description.Model;
import net.itimothy.rest.description.Primitive;
import net.itimothy.rest.description.Property;
import org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse;
import org.elasticsearch.cluster.metadata.MappingMetaData;
import org.elasticsearch.common.collect.ImmutableOpenMap;
import org.elasticsearch.common.hppc.cursors.ObjectCursor;

import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import static java.util.Arrays.asList;

public class ModelsCatalog {

    private final DataProvider dataProvider;
    private String indexOrAlias;

    Map<String, List<Model>> indexTypeModelsMap;
    Map<String, List<Model>> indexDocumentModelsMap;

    public ModelsCatalog(DataProvider dataProvider, String indexOrAlias) {
        this.dataProvider = dataProvider;
        this.indexOrAlias = indexOrAlias;
    }

    public Map<String, List<Model>> getIndexTypeModelsMap() {
        if (indexTypeModelsMap == null) {
            indexTypeModelsMap = new HashMap<>();

            GetMappingsResponse getMappingsResponse = dataProvider.getClient().admin().indices().prepareGetMappings().get();

            ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> indexTypeMappings = getMappingsResponse.getMappings();
            for (ObjectCursor<String> indexCursor : indexTypeMappings.keys()) {
                String indexName = indexCursor.value;
                List<Model> typeModels = new ArrayList<>();
                indexTypeModelsMap.put(indexName, typeModels);

                ImmutableOpenMap<String, MappingMetaData> typeMappings = indexTypeMappings.get(indexName);
                for (ObjectCursor<String> typeCursor : typeMappings.keys()) {
                    String typeName = typeCursor.value;

                    Map mappingProperties = null;
                    try {
                        mappingProperties = (Map) typeMappings.get(typeCursor.value).getSourceAsMap().get("properties");
                    } catch (IOException e) {
                        e.printStackTrace();
                    }

                    List<Property> properties = new ArrayList<>();
                    for (Object propertyNameObj : mappingProperties.keySet()) {
                        String propertyName = propertyNameObj.toString();
                        if (mappingProperties.containsKey(propertyName)) {
                            Map fieldMapping = (Map) mappingProperties.get(propertyName);
                            String mappingType = fieldMapping.containsKey("type") ? fieldMapping.get("type").toString() : "string";
                            properties.add(
                                Property.builder()
                                    .name(propertyName.toString())
                                    .model(mappingType != null ? mappingTypeToModel(mappingType) : null).build()
                            );
                        }
                    }

                    typeModels.add(
                        Model.builder()
                            .id(getTypeModelId(indexName, typeName))
                            .name(typeName)
                            .properties(properties)
                            .build()
                    );
                }
            }
        }

        return indexTypeModelsMap;
    }

    public Map<String, List<Model>> getIndexDocumentModelsMap() {
        if (indexDocumentModelsMap == null) {
            indexDocumentModelsMap = getIndexTypeModelsMap().entrySet().stream()
                .collect(
                    Collectors.toMap(
                        e -> e.getKey(),
                        e -> {
                            String indexName = e.getKey();
                            List<Model> typeModels = e.getValue();

                            return typeModels.stream()
{'fraction_non_alphanumeric': 0.09227028410545175, 'fraction_numerical': 0.0, 'mean_word_length': 1.8710506980161645, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1791208', 'n_tokens_mistral': 1848, 'n_tokens_neox': 1787, 'n_words': 334}