<commit_before>/**
 * @author <NAME>
 * @author <NAME>
 */

var elasticsearch = require('elasticsearch');
var Promise = require('bluebird');
var types = require('./config/application').types;

var indexName = require('./config/application').indexName;
var esAddress = require('./config/application').esAddress

var mappings = {};
types.forEach(function(type) {
	var path = './config/types/' + type + '/mapping';
	mappings[type] = require(path);
});

var client = new elasticsearch.Client({
	host: esAddress,
	log: 'info'
});

/**
 * Update mappings for all types
 *
 * @param index
 * @param callback
 */
function updateMappings(index, callback) {
    var promises = [];
    types.forEach(function(type) {

        console.log("type: "+type+" updated ");
            // ":"+JSON.stringify(mappings[type]))

        promises.push(client.indices.putMapping({
            index: index, type: type, body: mappings[type]
        }));
    });
    Promise.all(promises).then(function(res) {
        callback(null, res);
    }).catch(function(err) {
        callback(err, null)
    });
}


/**
 * Reindex current index documents to new index
 *
 * @param sourceIndex
 * @param targetIndex
 * @param lastIndexTime
 * @param callback
 */
function copyIndex(sourceIndex, targetIndex, lastIndexTime, callback) {

    var currentIndexTime = new Date();

    var esReq = {
        index: sourceIndex,
        scroll: '1m',

        // TODO REVIEW - REPLACED THE QUERY
        body: {
            query: {
                match_all: {}
            }
        }
    };
    console.log("Scroll query", JSON.stringify(esReq, null, 4));
    var count = 0;
    client.search(esReq, function getMoreUntilDone(err, res) {
        if (err) return callback(err, null);
        console.log("Scrolling", res);
        if (res.hits.total == 0) return callback(null, true);
        var bulk = [];
        res.hits.hits.forEach(function(hit) {
            bulk.push({ index:  { _index: targetIndex, _type: hit._type, _id: hit._id } });
            bulk.push(hit._source);
            count++;
        });
        client.bulk({ body: bulk }, function(err, bulkRes) {
            console.log("Bulk index result", bulkRes);
            if (err) return callback(err, null);
            if (res.hits.total > count) {
                client.scroll({ scrollId: res._scroll_id, scroll: '1m' }, getMoreUntilDone);
            } else {
                client.clearScroll({ scrollId: res._scroll_id });
                // repeform copy with new lastIndexTime to ensure new documents are copied

                // TODO REVIEW - THE NEXT LINE GOT REPLACED BY THE ONE FOLLOWING - SEE ALSO COMMENT ABOVE
                // copyIndex(sourceIndex, targetIndex, currentIndexTime, callback);
                return callback(null,true);
            }
        });
    });
}

/**
 * Get index to which alias currently points
 */
function retrieveCurrentIndex(alias, callback) {
    client.indices.getAlias({ name: alias }, function(err, res) {
        if (err) return callback(err, null);
        var indices = Object.keys(res);
        var currentIndex = indexName + "_1";
        if (indices.length == 1) {
            var currentIndex = indices[0];
        }
        callback(null, currentIndex);
    });
}

/**
 * Determines if only the index named indexName does exist and no prefixed
 * versions of it, which is the only condition an initial update is allowed
 *
 * @param callback(updateCondition)
 *   initialUpdateAllowed is boolean can have one of the values
 *     false
 *     true
 */
var determineInitialUpdateCondition = function(callback){<commit_msg>Fix comment.<commit_after>/**
 * @author <NAME>
 * @author <NAME>
 */

var elasticsearch = require('elasticsearch');
var Promise = require('bluebird');
var types = require('./config/application').types;

var indexName = require('./config/application').indexName;
var esAddress = require('./config/application').esAddress

var mappings = {};
types.forEach(function(type) {
	var path = './config/types/' + type + '/mapping';
	mappings[type] = require(path);
});

var client = new elasticsearch.Client({
	host: esAddress,
	log: 'info'
});

/**
 * Update mappings for all types
 *
 * @param index
 * @param callback
 */
function updateMappings(index, callback) {
    var promises = [];
    types.forEach(function(type) {

        console.log("type: "+type+" updated ");
            // ":"+JSON.stringify(mappings[type]))

        promises.push(client.indices.putMapping({
            index: index, type: type, body: mappings[type]
        }));
    });
    Promise.all(promises).then(function(res) {
        callback(null, res);
    }).catch(function(err) {
        callback(err, null)
    });
}


/**
 * Reindex current index documents to new index
 *
 * @param sourceIndex
 * @param targetIndex
 * @param lastIndexTime
 * @param callback
 */
function copyIndex(sourceIndex, targetIndex, lastIndexTime, callback) {

    var currentIndexTime = new Date();

    var esReq = {
        index: sourceIndex,
        scroll: '1m',

        // TODO REVIEW - REPLACED THE QUERY
        body: {
            query: {
                match_all: {}
            }
        }
    };
    console.log("Scroll query", JSON.stringify(esReq, null, 4));
    var count = 0;
    client.search(esReq, function getMoreUntilDone(err, res) {
        if (err) return callback(err, null);
        console.log("Scrolling", res);
        if (res.hits.total == 0) return callback(null, true);
        var bulk = [];
        res.hits.hits.forEach(function(hit) {
            bulk.push({ index:  { _index: targetIndex, _type: hit._type, _id: hit._id } });
            bulk.push(hit._source);
            count++;
        });
        client.bulk({ body: bulk }, function(err, bulkRes) {
            console.log("Bulk index result", bulkRes);
            if (err) return callback(err, null);
            if (res.hits.total > count) {
                client.scroll({ scrollId: res._scroll_id, scroll: '1m' }, getMoreUntilDone);
            } else {
                client.clearScroll({ scrollId: res._scroll_id });
                // repeform copy with new lastIndexTime to ensure new documents are copied

                // TODO REVIEW - THE NEXT LINE GOT REPLACED BY THE ONE FOLLOWING - SEE ALSO COMMENT ABOVE
                // copyIndex(sourceIndex, targetIndex, currentIndexTime, callback);
                return callback(null,true);
            }
        });
    });
}

/**
 * Get index to which alias currently points
 */
function retrieveCurrentIndex(alias, callback) {
    client.indices.getAlias({ name: alias }, function(err, res) {
        if (err) return callback(err, null);
        var indices = Object.keys(res);
        var currentIndex = indexName + "_1";
        if (indices.length == 1) {
            var currentIndex = indices[0];
        }
        callback(null, currentIndex);
    });
}

/**
 * Determines if only the index named indexName does exist and no prefixed
 * versions of it, which is the only condition an initial update is allowed
 *
 * @param callback(initialUpdateAllowed)
 *   initialUpdateAllowed: boolean
 */
var determineInitialUpdateCondition = function(callback){
{'fraction_non_alphanumeric': 0.13255360623781676, 'fraction_numerical': 0.002227791701475912, 'mean_word_length': 2.4835111542192045, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2294695', 'n_tokens_mistral': 2057, 'n_tokens_neox': 2013, 'n_words': 553}