<commit_before>/*
 * Flush stats to ElasticSearch (http://www.elasticsearch.org/)
 *
 * To enable this backend, include 'elastic' in the backends
 * configuration array:
 *
 *   backends: ['./backends/elastic']
 *  (if the config file is in the statsd folder)
 *
 * A sample configuration can be found in exampleElasticConfig.js
 *
 * This backend supports the following config options:
 *
 *   host:          hostname or IP of ElasticSearch server
 *   port:          port of Elastic Search Server
 *   path:          http path of Elastic Search Server (default: '/')
 *   indexPrefix:   Prefix of the dynamic index to be created (default: 'statsd')
 *   indexType:     The dociment type of the saved stat (default: 'stat')
 */

var net = require('net'),
   util = require('util'),
   http = require('http');

var debug;
var flushInterval;
var elasticHost;
var elasticPort;
var elasticPath;
var elasticIndex;
var elasticCountType;
var elasticTimerType;

var elasticStats = {};


var es_bulk_insert = function elasticsearch_bulk_insert(listCounters, listTimers, listTimerData) {

      var indexDate = new Date();
      var indexMo = indexDate.getUTCMonth() +1;
      if (indexMo < 10) {
          indexMo = '0'+indexMo;
      }
      var indexDt = indexDate.getUTCDate();
      if (indexDt < 10) {
          indexDt = '0'+indexDt;
      }

      var statsdIndex = elasticIndex + '-' + indexDate.getUTCFullYear() + '.' + indexMo + '.' + indexDt;

      var payload = '';
      for (key in listCounters) {
        payload += '{"index":{"_index":"'+statsdIndex+'","_type":"'+elasticCountType+'"}}'+"\n";
        payload += '{';
        innerPayload = '';
          for (statKey in listCounters[key]){
            if (innerPayload) innerPayload += ',';

            innerPayload += '"'+statKey+'":'+writeValue(listCounters[key][statKey]);
          }
        payload += innerPayload +'}'+"\n";
      }
      for (key in listTimers) {
        payload += '{"index":{"_index":"'+statsdIndex+'","_type":"'+elasticTimerType+'"}}'+"\n";
        payload += '{';
        innerPayload = '';
          for (statKey in listTimers[key]){
            if (innerPayload) innerPayload += ',';
            innerPayload += '"'+statKey+'":'+writeValue(listTimers[key][statKey]);
          }
        payload += innerPayload +'}'+"\n";
      }
      for (key in listTimerData) {
        payload += '{"index":{"_index":"'+statsdIndex+'","_type":"'+elasticTimerType+'_stats"}}'+"\n";
        payload += '{';
        innerPayload = '';
          for (statKey in listTimerData[key]){
            if (innerPayload) innerPayload += ',';
            innerPayload += '"'+statKey+'":'+writeValue(listTimerData[key][statKey]);
          }
        payload += innerPayload +'}'+"\n";
      }


      var optionsPost = {
        host: elasticHost,
        port: elasticPort,
        path: elasticPath + statsdIndex + '/' + '/_bulk',
        method: 'POST',
        headers: {
          'Content-Type': 'application/json',
          'Content-Length': payload.length
        }
      };
      var req = http.request(optionsPost, function(res) {
          res.on('data', function(d) {
            if (Math.floor(res.statusCode / 100) == 5){
              var errdata = "HTTP " + res.statusCode + ": " + d;
              console.log(errdata);
            }
          });
      });

      if (debug) {
        console.log(payload);
      }
      req.write(payload);
      req.end();
}

function writeValue(value) {
  if (typeof value === 'object') {
    for(var propertyName in value) {
      return writeValue(value[propertyName]);
    }
  } else if (typeof value === 'string') {
    return '"'+value+'"';
  } else {
    return value;
  }
}

var flush_stats = function elastic_flush(ts, metrics) {
  console.log( JSON.stringify(metrics) );
  var statString = '';
  var numStats = 0;
  var key;
  var array_counts     = new Array();
  var array_timers     = new Array();
  var array_timer_data = new Array();

  ts = ts*1000;
/*
  var gauges = metrics.gauges;
  var pctThreshold = metrics.pctThreshold;
*/

  for (key in metrics.counters) {
    var listKeys = key.split('.');
    var value = metrics.counters[key];
    array_counts.push({
    "source": listKeys[0] || '',<commit_msg>Added debug line to metrics print<commit_after>/*
 * Flush stats to ElasticSearch (http://www.elasticsearch.org/)
 *
 * To enable this backend, include 'elastic' in the backends
 * configuration array:
 *
 *   backends: ['./backends/elastic']
 *  (if the config file is in the statsd folder)
 *
 * A sample configuration can be found in exampleElasticConfig.js
 *
 * This backend supports the following config options:
 *
 *   host:          hostname or IP of ElasticSearch server
 *   port:          port of Elastic Search Server
 *   path:          http path of Elastic Search Server (default: '/')
 *   indexPrefix:   Prefix of the dynamic index to be created (default: 'statsd')
 *   indexType:     The dociment type of the saved stat (default: 'stat')
 */

var net = require('net'),
   util = require('util'),
   http = require('http');

var debug;
var flushInterval;
var elasticHost;
var elasticPort;
var elasticPath;
var elasticIndex;
var elasticCountType;
var elasticTimerType;

var elasticStats = {};


var es_bulk_insert = function elasticsearch_bulk_insert(listCounters, listTimers, listTimerData) {

      var indexDate = new Date();
      var indexMo = indexDate.getUTCMonth() +1;
      if (indexMo < 10) {
          indexMo = '0'+indexMo;
      }
      var indexDt = indexDate.getUTCDate();
      if (indexDt < 10) {
          indexDt = '0'+indexDt;
      }

      var statsdIndex = elasticIndex + '-' + indexDate.getUTCFullYear() + '.' + indexMo + '.' + indexDt;

      var payload = '';
      for (key in listCounters) {
        payload += '{"index":{"_index":"'+statsdIndex+'","_type":"'+elasticCountType+'"}}'+"\n";
        payload += '{';
        innerPayload = '';
          for (statKey in listCounters[key]){
            if (innerPayload) innerPayload += ',';

            innerPayload += '"'+statKey+'":'+writeValue(listCounters[key][statKey]);
          }
        payload += innerPayload +'}'+"\n";
      }
      for (key in listTimers) {
        payload += '{"index":{"_index":"'+statsdIndex+'","_type":"'+elasticTimerType+'"}}'+"\n";
        payload += '{';
        innerPayload = '';
          for (statKey in listTimers[key]){
            if (innerPayload) innerPayload += ',';
            innerPayload += '"'+statKey+'":'+writeValue(listTimers[key][statKey]);
          }
        payload += innerPayload +'}'+"\n";
      }
      for (key in listTimerData) {
        payload += '{"index":{"_index":"'+statsdIndex+'","_type":"'+elasticTimerType+'_stats"}}'+"\n";
        payload += '{';
        innerPayload = '';
          for (statKey in listTimerData[key]){
            if (innerPayload) innerPayload += ',';
            innerPayload += '"'+statKey+'":'+writeValue(listTimerData[key][statKey]);
          }
        payload += innerPayload +'}'+"\n";
      }


      var optionsPost = {
        host: elasticHost,
        port: elasticPort,
        path: elasticPath + statsdIndex + '/' + '/_bulk',
        method: 'POST',
        headers: {
          'Content-Type': 'application/json',
          'Content-Length': payload.length
        }
      };
      var req = http.request(optionsPost, function(res) {
          res.on('data', function(d) {
            if (Math.floor(res.statusCode / 100) == 5){
              var errdata = "HTTP " + res.statusCode + ": " + d;
              console.log(errdata);
            }
          });
      });

      if (debug) {
        console.log(payload);
      }
      req.write(payload);
      req.end();
}

function writeValue(value) {
  if (typeof value === 'object') {
    for(var propertyName in value) {
      return writeValue(value[propertyName]);
    }
  } else if (typeof value === 'string') {
    return '"'+value+'"';
  } else {
    return value;
  }
}

var flush_stats = function elastic_flush(ts, metrics) {
  if (debug) {
    console.log( JSON.stringify(metrics) )
  };
  var statString = '';
  var numStats = 0;
  var key;
  var array_counts     = new Array();
  var array_timers     = new Array();
  var array_timer_data = new Array();

  ts = ts*1000;
/*
  var gauges = metrics.gauges;
  var pctThreshold = metrics.pctThreshold;
*/

  for (key in metrics.counters) {
    var listKeys = key.split('.');
    var value = metrics.counters[key];
    array_counts.push({
    "source": listKeys[0] || '',
{'fraction_non_alphanumeric': 0.15608496655322146, 'fraction_numerical': 0.003990142002112428, 'mean_word_length': 2.5244003308519436, 'pattern_counts': {'":': 28, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2089368', 'n_tokens_mistral': 2648, 'n_tokens_neox': 2587, 'n_words': 631}