<commit_before>'use strict'
function MultiLine (delimiter, cbf) {
  this.opt = {
    delimiter: delimiter,
    timeout: Number(process.env.LOGAGENT_MULTILINE_TIMEOUT_MS) || 300
  }
  this.lines = []
  this.state = 0
  this.consumer = cbf
  this.tid = 0
  if (delimiter) {
    this.tid = setInterval(function lineTimeout() {
      if (this.lines.length > 0 && (new Date().getTime() - this.lastCall.getTime()) > this.opt.timeout) {
        this.consumer(this.lines.join('\n'))
        this.lines.length = 0
        this.state = 0
      }
    }.bind(this), this.opt.timeut )
    if(this.tid.unref) {
      this.tid.unref()
    }
  }
}

MultiLine.prototype.intervalHandler = function () {}

MultiLine.prototype.add = function (line) {
  if (!this.opt.delimiter) {
    return this.consumer(line)
  }
  this.lastCall = new Date()
  if (this.lines.length === 0) {
    this.lines.push(line)
  } else { // reading in block
    if (this.opt.delimiter.test(line)) {
      this.consumer(this.lines.join('\n'))
      this.lines.length = 0
      this.lines.push(line)
    } else {
      this.lines.push(line)
    }
  }
}

module.exports = MultiLine<commit_msg>fix high cpu usage<commit_after>'use strict'
function MultiLine (delimiter, cbf) {
  this.opt = {
    delimiter: delimiter,
    timeout: Number(process.env.LOGAGENT_MULTILINE_TIMEOUT_MS) || 300
  }
  this.lines = []
  this.state = 0
  this.consumer = cbf
  this.tid = 0
  if (delimiter) {
    this.tid = setInterval(function lineTimeout() {
      if (this.lines.length > 0 && (new Date().getTime() - this.lastCall.getTime()) > this.opt.timeout) {
        this.consumer(this.lines.join('\n'))
        this.lines.length = 0
        this.state = 0
      }
    }.bind(this), Math.max (this.opt.timeout, 300))
    if(this.tid.unref) {
      this.tid.unref()
    }
  }
}

MultiLine.prototype.intervalHandler = function () {}

MultiLine.prototype.add = function (line) {
  if (!this.opt.delimiter) {
    return this.consumer(line)
  }
  this.lastCall = new Date()
  if (this.lines.length === 0) {
    this.lines.push(line)
  } else { // reading in block
    if (this.opt.delimiter.test(line)) {
      this.consumer(this.lines.join('\n'))
      this.lines.length = 0
      this.lines.push(line)
    } else {
      this.lines.push(line)
    }
  }
}

module.exports = MultiLine
{'fraction_non_alphanumeric': 0.15650670125378296, 'fraction_numerical': 0.009943795936013834, 'mean_word_length': 2.638364779874214, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5169150', 'n_tokens_mistral': 846, 'n_tokens_neox': 816, 'n_words': 144}