<commit_before>'use strict';
var Promise = require('bluebird');

var DOCUMENT_UPDATED = 200;
var DOCUMENT_CREATED = 201;
var DOCUMENT_EXISTS = 409;

module.exports = function(context, opConfig, connection) {
    var logger = context.logger;

    var client = context.foundation.getConnection({
            endpoint: connection,
            type: 'elasticsearch',
            cached: true
        }).client;

    var bulk_size = 1000;
    if (opConfig.bulk_size) bulk_size = opConfig.bulk_size;

    var activeBulk = [];
    var activeCount = 0;
    var runningBulk = 0;

    // We flush periodically to make sure we don't end up with data languishing
    // in a buffer that is not actively being written to.
    var cacheFlusher = setInterval(function() {
        if (activeBulk.length > 0) {
            logger.info("Flushing bulk cache.");

            var bulkSave = activeBulk;
            activeBulk = [];

            _flush(bulkSave);
        }
    }, 30000);

    function _flush(bulkSave) {

        return new Promise(function(resolve, reject) {
            runningBulk++;

            client.bulk({
                body: bulkSave
            }, function(err, resp) {
                activeCount += bulkSave.length / 2;

                //bulkSave = null;
                runningBulk--;

                if (err) {
                    logger.error("Detail save general error: " + err);
                }
                else {
                    resp.items.forEach(function(item, index) {
                        if (item.create) {<commit_msg>Disabling the background cache flushing on router (#281)<commit_after>'use strict';
var Promise = require('bluebird');

var DOCUMENT_UPDATED = 200;
var DOCUMENT_CREATED = 201;
var DOCUMENT_EXISTS = 409;

module.exports = function(context, opConfig, connection) {
    var logger = context.logger;

    var client = context.foundation.getConnection({
            endpoint: connection,
            type: 'elasticsearch',
            cached: true
        }).client;

    var bulk_size = 1000;
    if (opConfig.size) bulk_size = opConfig.size;

    var activeBulk = [];
    var activeCount = 0;
    var runningBulk = 0;

    // We flush periodically to make sure we don't end up with data languishing
    // in a buffer that is not actively being written to.
    /*var cacheFlusher = setInterval(function() {
        if (activeBulk.length > 0) {
            logger.info("Flushing bulk cache.");

            var bulkSave = activeBulk;
            activeBulk = [];

            _flush(bulkSave);
        }
    }, 30000);*/

    function _flush(bulkSave) {

        return new Promise(function(resolve, reject) {
            runningBulk++;

            client.bulk({
                body: bulkSave
            }, function(err, resp) {
                activeCount += bulkSave.length / 2;

                //bulkSave = null;
                runningBulk--;

                if (err) {
                    logger.error("Detail save general error: " + err);
                }
                else {
                    resp.items.forEach(function(item, index) {
                        if (item.create) {
{'fraction_non_alphanumeric': 0.10009532888465204, 'fraction_numerical': 0.014934858595487766, 'mean_word_length': 1.7517482517482517, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '927635', 'n_tokens_mistral': 925, 'n_tokens_neox': 828, 'n_words': 235}