<commit_before>package com.arquivolivre.elastiky.commons;

import com.arquivolivre.elastiky.annotations.Analyzer;
import com.arquivolivre.elastiky.annotations.Ignored;
import com.arquivolivre.elastiky.annotations.Index;
import com.arquivolivre.elastiky.annotations.Nested;
import com.arquivolivre.elastiky.annotations.NotAnalyzed;
import com.arquivolivre.elastiky.annotations.NotIndexed;
import static com.arquivolivre.elastiky.commons.Types.isBasicType;
import static com.arquivolivre.elastiky.commons.Types.isGeneric;
import com.google.gson.Gson;
import java.lang.annotation.Annotation;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Modifier;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.util.Map;
import java.util.TreeMap;
import org.apache.log4j.Logger;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.client.Client;
import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
import org.elasticsearch.common.settings.Settings;

/**
 *
 * @author <NAME> <<EMAIL>>
 */
public class IndexManagerImpl implements IndexManager {

    private final Client elasticSearchClient;
    private BulkRequestBuilder bulkRequest;
    private final Logger logger = Logger.getLogger(IndexManagerImpl.class);

    IndexManagerImpl(Client client) {
        this.elasticSearchClient = client;
    }

    public static IndexManagerImpl build(Client client) {
        return new IndexManagerImpl(client);
    }

    public Client getElasticSearchClient() {
        return elasticSearchClient;
    }

    @Override
    public void addToBulk(String id, Object source) {
        if (bulkRequest == null) {
            bulkRequest = elasticSearchClient.prepareBulk();
        }
        if (source != null) {
            String json = new Gson().toJson(source);
            if (!json.isEmpty()) {
                IndexInfo indexInfo = getIndexInfo(source.getClass());
                if (!indexExists(indexInfo.getName())) {
                    createIndex(indexInfo.getName(), indexInfo.getType(), source);
                }
                bulkRequest.add(elasticSearchClient.prepareIndex(indexInfo.getName(), indexInfo.getType(), id).setSource(json));
                logger.info(String.format("Object (id: %s) added to bulk", id));
            }
        } else {
            logger.warn("Attempting to add an empty object, ignoring...");
        }
    }

    @Override
    public void executeBulkAdd() {
        if (bulkRequest != null) {
            logger.info("Executing bulk add...");
            BulkResponse actionGet = bulkRequest.execute().actionGet();
            if (actionGet.hasFailures()) {
                logger.error(actionGet.buildFailureMessage());
            } else {
                logger.info("Done!");
            }
            bulkRequest = null;
        }
    }

    @Override
    public <A> A get(String id, Class<A> clazz) {
        IndexInfo indexInfo = getIndexInfo(clazz);
        GetResponse response = elasticSearchClient.prepareGet(indexInfo.getName(), indexInfo.getType(), id).execute().actionGet();
        if (response.isSourceEmpty()) {
            String msg = String.format("Object (id: %s) of the type %s was not found in index %s!", id, indexInfo.getName(), indexInfo.getType());
            logger.warn(msg);
            return null;
        }
        return new Gson().fromJson(response.getSourceAsString(), clazz);
    }

    private IndexInfo getIndexInfo(Class clazz) {
        try {
            Annotation annotation = clazz.getAnnotation(Index.class);
            String index = (String) annotation.annotationType().getMethod("name").invoke(annotation);
            String type = (String) annotation.annotationType().getMethod("type").invoke(annotation);
            return new IndexInfo(index, type);
        } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException ex) {
            logger.error("Error on retrieving index information.", ex);
        }
        return null;
    }

    @Override
    public void putMapping(Object o) {
        IndexInfo indexInfo = getIndexInfo(o.getClass());
        logger.info("puttin map to index " + indexInfo.getName());
        elasticSearchClient.admin().indices().preparePutMapping(indexInfo.getName())
                .setType(indexInfo.getType())
                .setSource(generateMapping(o))
                .execute()
                .actionGet();
    }

    @Override
    public boolean indexExists(String index) {
        IndicesExistsResponse response = elasticSearchClient.admin().indices().exists(new IndicesExistsRequest(index)).actionGet();
        return response.isExists();
    }

    @Override
    public void createIndex(String indexName, String indexType, Object source) {
        logger.info(String.format("Generating index %s ...", indexName));
        CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName);
        if (indexType != null) {
            String settings = generateSettings(source);
            if (settings != null) {
                logger.info("Setting up...");
                createIndexRequest.settings(settings);
            }
        }
        logger.info("Mapping...");
        String mapping = generateMapping(source);
        createIndexRequest.mapping(indexType, mapping);
        try {
            CreateIndexResponse response = elasticSearchClient.admin().indices().create(createIndexRequest).actionGet();
            if (response.isAcknowledged()) {
                logger.info(String.format("Index %s created!", indexName));
            }
        } catch (ElasticsearchException ex) {
            logger.error(String.format("Index %s was not created due some errors.", indexName), ex);
        }
    }

    public String generateMapping(Object obj) {
        Map<Object, Object> typeMap = new TreeMap<>();
        Map<Object, Object> properties = new TreeMap<>();
        IndexInfo indexInfo = getIndexInfo(obj.getClass());
        typeMap.put(indexInfo.getType(), properties);
        try {
            properties.put("properties", getFields(obj.getClass(), false));
        } catch (SecurityException | IllegalArgumentException | IllegalAccessException ex) {
            logger.error("Error while parsing mapping.", ex);
        }
        return new Gson().toJson(typeMap);
    }
<commit_msg>some improvements to settings<commit_after>package com.arquivolivre.elastiky.commons;

import com.arquivolivre.elastiky.annotations.Analyzer;
import com.arquivolivre.elastiky.annotations.Ignored;
import com.arquivolivre.elastiky.annotations.Index;
import com.arquivolivre.elastiky.annotations.Nested;
import com.arquivolivre.elastiky.annotations.NotAnalyzed;
import com.arquivolivre.elastiky.annotations.NotIndexed;
import static com.arquivolivre.elastiky.commons.Types.isBasicType;
import static com.arquivolivre.elastiky.commons.Types.isGeneric;
import com.google.gson.Gson;
import java.lang.annotation.Annotation;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Modifier;
import java.lang.reflect.ParameterizedType;
import java.lang.reflect.Type;
import java.util.Map;
import java.util.TreeMap;
import org.apache.log4j.Logger;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.get.GetResponse;
import org.elasticsearch.client.Client;
import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder;
import org.elasticsearch.common.settings.Settings;

/**
 *
 * @author <NAME> <<EMAIL>>
 */
public class IndexManagerImpl implements IndexManager {

    private final Client elasticSearchClient;
    private BulkRequestBuilder bulkRequest;
    private final Logger logger = Logger.getLogger(IndexManagerImpl.class);

    IndexManagerImpl(Client client) {
        this.elasticSearchClient = client;
    }

    public static IndexManagerImpl build(Client client) {
        return new IndexManagerImpl(client);
    }

    public Client getElasticSearchClient() {
        return elasticSearchClient;
    }

    @Override
    public void addToBulk(String id, Object source) {
        if (bulkRequest == null) {
            bulkRequest = elasticSearchClient.prepareBulk();
        }
        if (source != null) {
            String json = new Gson().toJson(source);
            if (!json.isEmpty()) {
                IndexInfo indexInfo = getIndexInfo(source.getClass());
                if (!indexExists(indexInfo.getName())) {
                    createIndex(indexInfo.getName(), indexInfo.getType(), source);
                }
                bulkRequest.add(elasticSearchClient.prepareIndex(indexInfo.getName(), indexInfo.getType(), id).setSource(json));
                logger.info(String.format("Object (id: %s) added to bulk", id));
            }
        } else {
            logger.warn("Attempting to add an empty object, ignoring...");
        }
    }

    @Override
    public void executeBulkAdd() {
        if (bulkRequest != null) {
            logger.info("Executing bulk add...");
            BulkResponse actionGet = bulkRequest.execute().actionGet();
            if (actionGet.hasFailures()) {
                logger.error(actionGet.buildFailureMessage());
            } else {
                logger.info("Done!");
            }
            bulkRequest = null;
        }
    }

    @Override
    public <A> A get(String id, Class<A> clazz) {
        IndexInfo indexInfo = getIndexInfo(clazz);
        GetResponse response = elasticSearchClient.prepareGet(indexInfo.getName(), indexInfo.getType(), id).execute().actionGet();
        if (response.isSourceEmpty()) {
            String msg = String.format("Object (id: %s) of the type %s was not found in index %s!", id, indexInfo.getName(), indexInfo.getType());
            logger.warn(msg);
            return null;
        }
        return new Gson().fromJson(response.getSourceAsString(), clazz);
    }

    private IndexInfo getIndexInfo(Class clazz) {
        try {
            Annotation annotation = clazz.getAnnotation(Index.class);
            String index = (String) annotation.annotationType().getMethod("name").invoke(annotation);
            String type = (String) annotation.annotationType().getMethod("type").invoke(annotation);
            return new IndexInfo(index, type);
        } catch (NoSuchMethodException | SecurityException | IllegalAccessException | IllegalArgumentException | InvocationTargetException ex) {
            logger.error("Error on retrieving index information.", ex);
        }
        return null;
    }

    @Override
    public void putMapping(Object o) {
        IndexInfo indexInfo = getIndexInfo(o.getClass());
        logger.info("puttin map to index " + indexInfo.getName());
        elasticSearchClient.admin().indices().preparePutMapping(indexInfo.getName())
                .setType(indexInfo.getType())
                .setSource(generateMapping(o))
                .execute()
                .actionGet();
    }

    @Override
    public boolean indexExists(String index) {
        IndicesExistsResponse response = elasticSearchClient.admin().indices().exists(new IndicesExistsRequest(index)).actionGet();
        return response.isExists();
    }

    @Override
    public void createIndex(String indexName, String indexType, Object source) {
        logger.info(String.format("Generating index %s ...", indexName));
        CreateIndexRequest createIndexRequest = new CreateIndexRequest(indexName);
        if (indexType != null) {
            String settings = generateSettings(source);
            if (settings != null) {
                logger.info("Setting up...");
                createIndexRequest.settings(settings);
            }
        }
        logger.info("Mapping...");
        String mapping = generateMapping(source);
        createIndexRequest.mapping(indexType, mapping);
        try {
            CreateIndexResponse response = elasticSearchClient.admin().indices().create(createIndexRequest).actionGet();
            if (response.isAcknowledged()) {
                logger.info(String.format("Index %s created!", indexName));
            }
        } catch (ElasticsearchException ex) {
            logger.error(String.format("Index %s was not created due some errors.", indexName), ex);
        }
    }

    public String generateMapping(Object obj) {
        Map<Object, Object> typeMap = new TreeMap<>();
        Map<Object, Object> properties = new TreeMap<>();
        IndexInfo indexInfo = getIndexInfo(obj.getClass());
        typeMap.put(indexInfo.getType(), properties);
        try {
            properties.put("properties", getFields(obj.getClass(), false));
        } catch (SecurityException | IllegalArgumentException | IllegalAccessException ex) {
            logger.error("Error while parsing mapping.", ex);
            return null;
        }
        return new Gson().toJson(typeMap);
    }

{'fraction_non_alphanumeric': 0.11061597067490836, 'fraction_numerical': 0.0001437504492201538, 'mean_word_length': 2.9641025641025642, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1566156', 'n_tokens_mistral': 3661, 'n_tokens_neox': 3626, 'n_words': 741}