<commit_before>var fhComponentMetrics = require('fh-component-metrics');
var syncUtils = require('./util');
var metricsClient = fhComponentMetrics({enabled: false});
var _ = require('underscore');
var async = require('async');

var MB = 1024*1024;

var METRIC_KEYS = {
  WORKER_JOB_ERROR_COUNT: "worker-get-job-error-count",
  WORKER_JOB_TOTAL_COUNT: "worker-job-count",
  WORKER_JOB_FAILURE_COUNT: "worker-job-failure-count",
  WORKER_JOB_SUCCESS_COUNT: "worker-job-success-count",
  WORKER_JOB_PROCESS_TIME: "worker-job-process-time",
  QUEUE_OPERATION_TIME: "queue-operation-time",
  HANDLER_OPERATION_TIME: "sync-handler-operation-time",
  SYNC_SCHEDULER_CHECK_TIME: "sync-scheduler-check-time",
  SYNC_REQUEST_TOTAL_PROCESS_TIME: "sync-request-total-process-time",
  PENDING_CHANGE_PROCESS_TIME: "pending-change-process-time",
  SYNC_API_PROCESS_TIME: "sync-api-process-time",
  MONGODB_OPERATION_TIME: "mongodb-operation-time",
  WORKER_QUEUE_SIZE: "worker-queue-size"
};

var MAX_NUMBER = Number.MAX_SAFE_INTEGER || Number.MAX_VALUE;

var statsNamespace = 'fhsyncstats';
var metricsTitle = (process.env.FH_TITLE || 'fhsync') + '_stats';

var redisClient;
var debug = syncUtils.debug;

var Timer = function() {
  this.start = Date.now();
};

Timer.prototype.stop = function() {
  var end = Date.now();
  return end - this.start;
};

var timeAsyncFunc = function(metricKey, targetFn) {
  return function() {
    var args = [].slice.call(arguments);
    if (typeof args[args.length - 1] !== 'function') {
      debug('can not time the target function %s as last argument is not a function', targetFn.name);
    } else {
      var callback = args.pop();
      var timer = new Timer();
      args.push(function() {
        var timing = timer.stop();
        metricsClient.gauge(metricKey, {success: !arguments[0], fn: targetFn.name}, timing);
        return callback.apply(null, arguments);
      });
    }
    targetFn.apply(null, args);
  }
};

/**
 * Compute the max, min, current, average values from the records
 * @param {Object} metric some meta data about the metric
 * @param {Array} records
 */
var aggregateData = function(metric, records) {
  var returnValue = {message: 'no stats available', name: metric.displayName};
  if (records && records.length > 0) {
    returnValue = _.chain(records).map(function(recordStr) {
      return JSON.parse(recordStr);
    }).groupBy(function(record) {
      return record.tags[metric.groupByTag];
    }).reduce(function(reduced, groupRecords, groupKey) {
      var processedData = _.reduce(groupRecords, function(memo, groupRecord) {
        groupRecords = _.sortBy(groupRecords, 'ts');
        var value = groupRecord.fields[metric.valueField];<commit_msg>Prefix all metrics with a metricsKey<commit_after>var fhComponentMetrics = require('fh-component-metrics');
var syncUtils = require('./util');
var metricsClient = fhComponentMetrics({enabled: false});
var _ = require('underscore');
var async = require('async');

var MB = 1024*1024;

var METRIC_KEYS = {
  WORKER_JOB_ERROR_COUNT: "worker-get-job-error-count",
  WORKER_JOB_TOTAL_COUNT: "worker-job-count",
  WORKER_JOB_FAILURE_COUNT: "worker-job-failure-count",
  WORKER_JOB_SUCCESS_COUNT: "worker-job-success-count",
  WORKER_JOB_PROCESS_TIME: "worker-job-process-time",
  QUEUE_OPERATION_TIME: "queue-operation-time",
  HANDLER_OPERATION_TIME: "sync-handler-operation-time",
  SYNC_SCHEDULER_CHECK_TIME: "sync-scheduler-check-time",
  SYNC_REQUEST_TOTAL_PROCESS_TIME: "sync-request-total-process-time",
  PENDING_CHANGE_PROCESS_TIME: "pending-change-process-time",
  SYNC_API_PROCESS_TIME: "sync-api-process-time",
  MONGODB_OPERATION_TIME: "mongodb-operation-time",
  WORKER_QUEUE_SIZE: "worker-queue-size"
};

var MAX_NUMBER = Number.MAX_SAFE_INTEGER || Number.MAX_VALUE;

var statsNamespace = 'fhsyncstats';
var metricsTitle = (process.env.FH_TITLE || 'fhsync') + '_stats';

var redisClient;
var debug = syncUtils.debug;

var Timer = function() {
  this.start = Date.now();
};

Timer.prototype.stop = function() {
  var end = Date.now();
  return end - this.start;
};

var timeAsyncFunc = function(metricKey, targetFn) {
  return function() {
    var args = [].slice.call(arguments);
    if (typeof args[args.length - 1] !== 'function') {
      debug('can not time the target function %s as last argument is not a function', targetFn.name);
    } else {
      var callback = args.pop();
      var timer = new Timer();
      args.push(function() {
        var timing = timer.stop();
        var prefixedMetricKey = [metricsTitle, '_', metricKey].join('');
        metricsClient.gauge(prefixedMetricKey, {success: !arguments[0], fn: targetFn.name}, timing);
        return callback.apply(null, arguments);
      });
    }
    targetFn.apply(null, args);
  }
};

/**
 * Compute the max, min, current, average values from the records
 * @param {Object} metric some meta data about the metric
 * @param {Array} records
 */
var aggregateData = function(metric, records) {
  var returnValue = {message: 'no stats available', name: metric.displayName};
  if (records && records.length > 0) {
    returnValue = _.chain(records).map(function(recordStr) {
      return JSON.parse(recordStr);
    }).groupBy(function(record) {
      return record.tags[metric.groupByTag];
    }).reduce(function(reduced, groupRecords, groupKey) {
      var processedData = _.reduce(groupRecords, function(memo, groupRecord) {
        groupRecords = _.sortBy(groupRecords, 'ts');
        var value = groupRecord.fields[metric.valueField];
{'fraction_non_alphanumeric': 0.1370880115900036, 'fraction_numerical': 0.00398406374501992, 'mean_word_length': 4.39882697947214, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6115116', 'n_tokens_mistral': 1825, 'n_tokens_neox': 1770, 'n_words': 382}