<commit_before>/// <reference path='../../../third_party/freedom-typings/console.d.ts' />
/// <reference path='../../../third_party/freedom-typings/freedom-common.d.ts' />
/// <reference path='../../../third_party/freedom-typings/freedom-module-env.d.ts' />

import logging = require('loggingprovider.types');

// The freedom console provider.
export var freedomConsole :freedom_Console.Console = freedom['core.console']();

// Besides output to console, log can also be buffered for later retrieval
// through "getLogs". This is the maximum number of buffered log before it is
// trimmed. Assuming average log length is 80, the whole buffer size is about
// 80k. That should be easy to send through email, not much memory usage, and
// still enough to capture most issues.
export var MAX_BUFFERED_LOG = 1000;

// Logs waiting for the logger to exist.
export var logBuffer: logging.Message[] = [];

// TODO: we probably will change it to false as default.
export var enabled = true;

// This represents a possible destination for log messages.  To make use of
// this, the class should be inherited from and the log_ method reimplemented
// to record the message in whichever way is best for that transport.
export class AbstractLoggingDestination {
  // These filters control what is displayed/saved for the different log types.
  // Entries for each type should be of the form:
  //   'tag': LEVEL
  // where LEVEL is the minimum level of log that will be processed for the
  // module 'tag'.  '*' is a wildcard tag that applies to any message that is not
  // specifically specified
  private filters_ :{[tag :string] :string} = {};

  constructor(filters :{[tag :string] :string}) {
    this.filters_ = filters;
  }

  private checkFilter_ = (level :string, tag :string) => {
    if (tag in this.filters_) {
      return isLevelAllowed_(level, this.filters_[tag]);
    }

    return '*' in this.filters_ && isLevelAllowed_(level, this.filters_['*']);
  }

  protected log_ = (level :string, tag :string, message :logging.Message) :void => {
    throw Error('not implemented');
  }

  public log = (level :string, tag :string, message :logging.Message) :void => {
    if (this.checkFilter_(level, tag)) {
      this.log_(level, tag, message);
    }
  }

  public setFilter = (args :string[]) => {
    this.filters_ = {};
    var parts :string[];

    for (var i in args) {
      parts = args[i].split(':');
      this.filters_[parts[0]] = parts[1];
    }
  }
}

// A logging destination for printing the message directly to the console
export class ConsoleLoggingDestination extends AbstractLoggingDestination {
  constructor() {
    super({'*': 'D'});
  }

  protected log_ = (level :string, tag :string, message :logging.Message) :void => {
    if (level === 'D') {
      freedomConsole.debug(tag, formatMessage(message));
    } else if (level === 'I') {
      freedomConsole.log(tag, formatMessage(message));
    } else if (level === 'W') {
      freedomConsole.warn(tag, formatMessage(message));
    } else {
      freedomConsole.error(tag, formatMessage(message));
    }
  }
}

export class BufferedLoggingDestination extends AbstractLoggingDestination {
  constructor() {
    super({'*': 'E'});
  }

  protected log_ = (level :string, tag :string, message :logging.Message) :void => {
    if (logBuffer.length > MAX_BUFFERED_LOG) {
      logBuffer.splice(0, MAX_BUFFERED_LOG / 10);
    }
    logBuffer.push(message);
  }
}

export var loggingDestinations :{[name :string] :AbstractLoggingDestination} = {};
var logDestination = {
  CONSOLE: 'console',
  BUFFERED: 'buffered'
};
loggingDestinations[logDestination.BUFFERED] =
  new BufferedLoggingDestination();
loggingDestinations[logDestination.CONSOLE] =
  new ConsoleLoggingDestination();
<commit_msg>let INFO messages be INFO messages<commit_after>/// <reference path='../../../third_party/freedom-typings/console.d.ts' />
/// <reference path='../../../third_party/freedom-typings/freedom-common.d.ts' />
/// <reference path='../../../third_party/freedom-typings/freedom-module-env.d.ts' />

import logging = require('loggingprovider.types');

// The freedom console provider.
export var freedomConsole :freedom_Console.Console = freedom['core.console']();

// Besides output to console, log can also be buffered for later retrieval
// through "getLogs". This is the maximum number of buffered log before it is
// trimmed. Assuming average log length is 80, the whole buffer size is about
// 80k. That should be easy to send through email, not much memory usage, and
// still enough to capture most issues.
export var MAX_BUFFERED_LOG = 1000;

// Logs waiting for the logger to exist.
export var logBuffer: logging.Message[] = [];

// TODO: we probably will change it to false as default.
export var enabled = true;

// This represents a possible destination for log messages.  To make use of
// this, the class should be inherited from and the log_ method reimplemented
// to record the message in whichever way is best for that transport.
export class AbstractLoggingDestination {
  // These filters control what is displayed/saved for the different log types.
  // Entries for each type should be of the form:
  //   'tag': LEVEL
  // where LEVEL is the minimum level of log that will be processed for the
  // module 'tag'.  '*' is a wildcard tag that applies to any message that is not
  // specifically specified
  private filters_ :{[tag :string] :string} = {};

  constructor(filters :{[tag :string] :string}) {
    this.filters_ = filters;
  }

  private checkFilter_ = (level :string, tag :string) => {
    if (tag in this.filters_) {
      return isLevelAllowed_(level, this.filters_[tag]);
    }

    return '*' in this.filters_ && isLevelAllowed_(level, this.filters_['*']);
  }

  protected log_ = (level :string, tag :string, message :logging.Message) :void => {
    throw Error('not implemented');
  }

  public log = (level :string, tag :string, message :logging.Message) :void => {
    if (this.checkFilter_(level, tag)) {
      this.log_(level, tag, message);
    }
  }

  public setFilter = (args :string[]) => {
    this.filters_ = {};
    var parts :string[];

    for (var i in args) {
      parts = args[i].split(':');
      this.filters_[parts[0]] = parts[1];
    }
  }
}

// A logging destination for printing the message directly to the console
export class ConsoleLoggingDestination extends AbstractLoggingDestination {
  constructor() {
    super({'*': 'D'});
  }

  protected log_ = (level :string, tag :string, message :logging.Message) :void => {
    if (level === 'D') {
      freedomConsole.debug(tag, formatMessage(message));
    } else if (level === 'I') {
      freedomConsole.info(tag, formatMessage(message));
    } else if (level === 'W') {
      freedomConsole.warn(tag, formatMessage(message));
    } else {
      freedomConsole.error(tag, formatMessage(message));
    }
  }
}

export class BufferedLoggingDestination extends AbstractLoggingDestination {
  constructor() {
    super({'*': 'E'});
  }

  protected log_ = (level :string, tag :string, message :logging.Message) :void => {
    if (logBuffer.length > MAX_BUFFERED_LOG) {
      logBuffer.splice(0, MAX_BUFFERED_LOG / 10);
    }
    logBuffer.push(message);
  }
}

export var loggingDestinations :{[name :string] :AbstractLoggingDestination} = {};
var logDestination = {
  CONSOLE: 'console',
  BUFFERED: 'buffered'
};
loggingDestinations[logDestination.BUFFERED] =
  new BufferedLoggingDestination();
loggingDestinations[logDestination.CONSOLE] =
  new ConsoleLoggingDestination();

{'fraction_non_alphanumeric': 0.13116057233704292, 'fraction_numerical': 0.003444621091679915, 'mean_word_length': 3.619951040391677, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '909821', 'n_tokens_mistral': 2212, 'n_tokens_neox': 2189, 'n_words': 725}