<commit_before>let debug = require('debug')('taskcluster-lib-monitor');
let _ = require('lodash');
let assert = require('assert');
let Promise = require('promise');
let taskcluster = require('taskcluster-client');
let raven = require('raven');
let Statsum = require('statsum');

class Monitor {

  constructor (authClient, sentry, statsumClient, opts) {
    this._opts = opts;
    this._auth = authClient;
    this._sentry = sentry; // This must be a Promise that resolves to {client, expires}
    this._statsum = statsumClient;

    if (!opts.isPrefixed && opts.reportStatsumErrors) {
      this._statsum.on('error', err => this.reportError(err, 'warning'));
    }

    if (!opts.isPrefixed && opts.patchGlobal) {
      process.on('uncaughtException', (err) => {
        console.log(err.stack);
        this.reportError(err);
        process.exit(1);
      });
      process.on('unhandledRejection', (reason, p) => {
        let err = 'Unhandled Rejection at: Promise ' + p + ' reason: ' + reason;
        console.log(err);
        this.reportError(err, 'warning');
      });
    }
  }

  async reportError (err, level='error') {
    this._sentry = this._sentry.then(async (sentry) => {
      if (!sentry.expires || Date.parse(sentry.expires) <= Date.now()) {
        let sentryInfo = await this._auth.sentryDSN(this._opts.project);
        return {
          client: new raven.Client(sentryInfo.dsn.secret),
          expires: sentryInfo.expires,
        };
      }
      return sentry;
    }).catch(err => {});

    this._sentry.then(sentry => {
      sentry.client.captureException(err, {level});
    });
  }

  // captureError is an alias for reportError to match up
  // with the raven api better.
  async captureError (err, level='error') {
    this.reportError(err, level);
  }

  count (key, val) {
    this._statsum.count(key, val || 1);
  }

  measure (key, val) {
    this._statsum.measure(key, val);
  }

  async flush () {
    await this._statsum.flush();
  }

  prefix (prefix) {
    let newopts = _.cloneDeep(this._opts);
    newopts.isPrefixed = true;
    return new Monitor(
      this._auth,
      this._sentry,
      this._statsum.prefix(prefix),
      newopts
    );
  }
}

class MockMonitor {
  constructor (opts, counts = {}, measures = {}, errors = []) {
    this._opts = opts;
    this.counts = counts;
    this.measures = measures;
    this.errors = errors;
  }

  async reportError (err, level='error') {
    this.errors.push(err);
  }

  async captureError (err, level='error') {
    this.reportError(err, level);
  }

  count (key, val) {
    let k = this._key(key);
    this.counts[k] = (this.counts[k] || 0) + (val || 1);
  }
<commit_msg>Add basic timers for standard tc needs<commit_after>let debug = require('debug')('taskcluster-lib-monitor');
let _ = require('lodash');
let assert = require('assert');
let Promise = require('promise');
let taskcluster = require('taskcluster-client');
let raven = require('raven');
let Statsum = require('statsum');

class Monitor {

  constructor (authClient, sentry, statsumClient, opts) {
    this._opts = opts;
    this._auth = authClient;
    this._sentry = sentry; // This must be a Promise that resolves to {client, expires}
    this._statsum = statsumClient;

    if (!opts.isPrefixed && opts.reportStatsumErrors) {
      this._statsum.on('error', err => this.reportError(err, 'warning'));
    }

    if (!opts.isPrefixed && opts.patchGlobal) {
      process.on('uncaughtException', (err) => {
        console.log(err.stack);
        this.reportError(err);
        process.exit(1);
      });
      process.on('unhandledRejection', (reason, p) => {
        let err = 'Unhandled Rejection at: Promise ' + p + ' reason: ' + reason;
        console.log(err);
        this.reportError(err, 'warning');
      });
    }
  }

  async reportError (err, level='error') {
    this._sentry = this._sentry.then(async (sentry) => {
      if (!sentry.expires || Date.parse(sentry.expires) <= Date.now()) {
        let sentryInfo = await this._auth.sentryDSN(this._opts.project);
        return {
          client: new raven.Client(sentryInfo.dsn.secret),
          expires: sentryInfo.expires,
        };
      }
      return sentry;
    }).catch(err => {});

    this._sentry.then(sentry => {
      sentry.client.captureException(err, {level});
    });
  }

  // captureError is an alias for reportError to match up
  // with the raven api better.
  async captureError (err, level='error') {
    this.reportError(err, level);
  }

  count (key, val) {
    this._statsum.count(key, val || 1);
  }

  measure (key, val) {
    this._statsum.measure(key, val);
  }

  async flush () {
    await this._statsum.flush();
  }

  prefix (prefix) {
    let newopts = _.cloneDeep(this._opts);
    newopts.isPrefixed = true;
    return new Monitor(
      this._auth,
      this._sentry,
      this._statsum.prefix(prefix),
      newopts
    );
  }

  // Given a function that operates on a
  // single message, this will time it and
  // report to statsum.
  timedHandler(name, handler) {
    return async (message) => {
      let start = process.hrtime();
      let success = 'success';
      try {
        await handler(message);
      } catch (e) {
        success = 'failure';
        throw e;
      } finally {
        let d = process.hrtime(start);
        this.measure(name + '.' + success, d[0] * 1000 + (d[1] / 1000000));
        this.measure(name + '.all', d[0] * 1000 + (d[1] / 1000000));
      }
    }
  }

  // Given an express api method, this will time it
  // and report to statsum.
  expressMiddleware(method) {
    return (req, res, next) => {
      let sent = false;
      let start = process.hrtime();
      let send = () => {
        try {
          // Avoid sending twice
          if (sent) {
            return;
          }
          sent = true;

          let d = process.hrtime(start);

          let success = 'success';
          if (res.statusCode >= 500) {
            success = 'server-error';
          } else if (res.statusCode >= 400) {
            success = 'client-error';
          }

          for (let stat of [success, 'all']) {
            let k = [method, stat].join('.');
            monitor.measure(k, d[0] * 1000 + (d[1] / 1000000));
            monitor.count(k);
          }
        } catch (e) {
          debug("Error while compiling response times: %s, %j", err, err, err.stack);
        }
      };
      res.once('finish', send);
      res.once('close', send);
      next();
    };
  }
}

class MockMonitor {
  constructor (opts, counts = {}, measures = {}, errors = []) {
    this._opts = opts;
    this.counts = counts;
    this.measures = measures;
    this.errors = errors;
  }

  async reportError (err, level='error') {
    this.errors.push(err);
  }

  async captureError (err, level='error') {
    this.reportError(err, level);
  }

  count (key, val) {
    let k = this._key(key);
    this.counts[k] = (this.counts[k] || 0) + (val || 1);
  }

{'fraction_non_alphanumeric': 0.15362485615650173, 'fraction_numerical': 0.007623705408515535, 'mean_word_length': 2.4251231527093595, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3165754', 'n_tokens_mistral': 2296, 'n_tokens_neox': 2220, 'n_words': 520}