<commit_before>var loghose = require('docker-loghose')
var through = require('through2')
var Logsene = require('logsene-js')
var SpmAgent = require('spm-agent')
var Logagent = require('logagent-js')
var fs = require('fs')

function createLogAgent () {
  var patternFile = '/etc/logagent/patterns.yml'
  if (fs.existsSync(patternFile)) {
    return new Logagent(patternFile)
  } else {
    console.log('No pattern file for log parsing found ' + patternFile + ' -> using default patterns')
    console.log('Use -v /mypattern/patterns.yml:' + patternFile + ' for custom log parser definitions.')
    return new Logagent() // use default patterns
  }
}
function extend (origin, add) {
  // Don't do anything if add isn't an object
  if (!add || typeof add !== 'object') return origin
  var keys = Object.keys(add)
  var i = keys.length
  while (i--) {
    origin[keys[i]] = add[keys[i]]
  }
  return origin
}

function DockerLogsene () {
  this.imageNameRegEx = /sematext\/spm-agent-docker/
  this.logagent = createLogAgent()
  this.logger = new Logsene(process.env.LOGSENE_TOKEN, 'docker')
  this.logger.on('error', function (err) {
    SpmAgent.Logger.log('error', 'Error in logsene-js: ' + err, err)
  })
  this.opts = {
    json: false,
    docker: null,
    events: null,
    // the following options limit the containers being matched
    // so we can avoid catching logs for unwanted containers
    matchByName: process.env.MATCH_BY_NAME,
    matchByImage: process.env.MATCH_BY_IMAGE,
    skipByName: process.env.SKIP_BY_NAME,
    skipByImage: process.env.SKIP_BY_IMAGE
  }
  var self = this
  setTimeout(function () {
    self.connect()
  }, 200)
}

DockerLogsene.prototype.debugLog = function (message)
{
  if (process.env.DEBUG_SPM_LOGGING === 'enabled') {
    SpmAgent.Logger.log('debug', message)
  }
}
DockerLogsene.prototype.getLogObject = function (line, container, cbf) {
  if (process.env.DEBUG_SPM_LOGGING==='enabled' && this.imageNameRegEx.test(container))
  {
    cbf('Disabled logging for image sematext/spm-agent-docker to avoid endless logging loop', null)
    return
  }
  var lines = line.split('\n')
  this.debugLog('Logs from Docker API (' + lines.length + ' lines)')
  this.debugLog('Logs from Docker API (content): ' + lines)
  var self = this
  for (var i = 0; i < lines.length; i++) {
    this.logagent.parseLine(line, container, function (err, parsed) {
      self.debugLog('Log parser result: ' + JSON.stringify(parsed))
      return cbf(err, parsed)
    })
  }
}

DockerLogsene.prototype.connect = function () {
  var self = this
  var options = self.opts
  SpmAgent.Logger.log('debug', 'connect logStream to docker.sock')
  self.logStream = through.obj(function (chunk, enc, cb) {
    try {
      if (!chunk.line) {
        cb()
        return
      }
      var lines = chunk.line.split('\n') // We can get multiple lines per chunk
      lines.forEach(function (line) {
        self.logLine(line, chunk)
      })
      cb()
    } catch (ex) {
      console.log(ex)
      cb()
    }
  })
  // self.logStream.once('error', this.reconnect)
  var lh = loghose(options)
  lh.pipe(self.logStream)
  lh.on('error', this.reconnect)
}

DockerLogsene.prototype.logLine = function (logline, data, next) {
  var self = this
  var lineParser = {
    chunk: data,
    line: logline,
    parse: function () {
      var chunk = this.chunk
      self.getLogObject(
      this.line,
      chunk.image + '_' + chunk.name + '_' + chunk.id,
      function (err, logObject) {
        if (err && !logObject) {
          return
        }
        if (logObject) {
          logObject.container_id = chunk.id
          logObject.image_name = chunk.image
          logObject.container_name = chunk.name
          logObject['@source'] = chunk.id + '/' + chunk.image
          var level = 'info'
          if (/error|fail|exception/i.test(this.line || '')) {
            level = 'error'
          }
          self.logger.log(new String( (logObject.level||logObject.lvl||logObject.severity||level)), logObject.message, logObject)
        }<commit_msg>optional removal of ANSI terminal colors using -e REMOVE_ANSI<commit_after>var loghose = require('docker-loghose')
var through = require('through2')
var Logsene = require('logsene-js')
var SpmAgent = require('spm-agent')
var Logagent = require('logagent-js')
var fs = require('fs')

function createLogAgent () {
  var patternFile = '/etc/logagent/patterns.yml'
  if (fs.existsSync(patternFile)) {
    return new Logagent(patternFile)
  } else {
    console.log('No pattern file for log parsing found ' + patternFile + ' -> using default patterns')
    console.log('Use -v /mypattern/patterns.yml:' + patternFile + ' for custom log parser definitions.')
    return new Logagent() // use default patterns
  }
}
function extend (origin, add) {
  // Don't do anything if add isn't an object
  if (!add || typeof add !== 'object') return origin
  var keys = Object.keys(add)
  var i = keys.length
  while (i--) {
    origin[keys[i]] = add[keys[i]]
  }
  return origin
}

function DockerLogsene () {
  this.imageNameRegEx = /sematext\/spm-agent-docker/
  this.logagent = createLogAgent()
  this.logger = new Logsene(process.env.LOGSENE_TOKEN, 'docker')
  this.logger.on('error', function (err) {
    SpmAgent.Logger.log('error', 'Error in logsene-js: ' + err, err)
  })
  this.opts = {
    json: false,
    docker: null,
    events: null,
    // the following options limit the containers being matched
    // so we can avoid catching logs for unwanted containers
    matchByName: process.env.MATCH_BY_NAME,
    matchByImage: process.env.MATCH_BY_IMAGE,
    skipByName: process.env.SKIP_BY_NAME,
    skipByImage: process.env.SKIP_BY_IMAGE
  }
  var self = this
  setTimeout(function () {
    self.connect()
  }, 200)
}

DockerLogsene.prototype.debugLog = function (message)
{
  if (process.env.DEBUG_SPM_LOGGING === 'enabled') {
    SpmAgent.Logger.log('debug', message)
  }
}
DockerLogsene.prototype.getLogObject = function (line, container, cbf) {
  if (process.env.DEBUG_SPM_LOGGING==='enabled' && this.imageNameRegEx.test(container))
  {
    cbf('Disabled logging for image sematext/spm-agent-docker to avoid endless logging loop', null)
    return
  }
  var lines = line.split('\n')
  this.debugLog('Logs from Docker API (' + lines.length + ' lines) ' + container)
  this.debugLog('Logs from Docker API (content) for : ' + container + '\n' + lines)
  var self = this
  for (var i = 0; i < lines.length; i++) {
    this.logagent.parseLine(line, container, function (err, parsed) {
      self.debugLog('Log parser result: ' + JSON.stringify(parsed))
      return cbf(err, parsed)
    })
  }
}

DockerLogsene.prototype.connect = function () {
  var self = this
  var options = self.opts
  SpmAgent.Logger.log('debug', 'connect logStream to docker.sock')
  self.logStream = through.obj(function (chunk, enc, cb) {
    try {
      if (!chunk.line) {
        cb()
        return
      }
      var lines = chunk.line.split('\n') // We can get multiple lines per chunk
      lines.forEach(function (line) {
        self.logLine(line, chunk)
      })
      cb()
    } catch (ex) {
      console.log(ex)
      cb()
    }
  })
  // self.logStream.once('error', this.reconnect)
  var lh = loghose(options)
  lh.pipe(self.logStream)
  lh.on('error', this.reconnect)
}

DockerLogsene.prototype.logLine = function (logline, data, next) {
  var self = this
  var lineParser = {
    chunk: data,
    line: process.env.REMOVE_ANSI ? logline.replace(
              /[\u001b\u009b][[()#;?]*(?:[0-9]{1,4}(?:;[0-9]{0,4})*)?[0-9A-ORZcf-nqry=><]/g,
              '') : logline,
    parse: function () {
      var chunk = this.chunk
      self.getLogObject(
      this.line,
      chunk.image + '_' + chunk.name + '_' + chunk.id,
      function (err, logObject) {
        if (err && !logObject) {
          return
        }
        if (logObject) {
          logObject.container_id = chunk.id
          logObject.image_name = chunk.image
          logObject.container_name = chunk.name
          logObject['@source'] = chunk.id + '/' + chunk.image
          var level = 'info'
          if (/error|fail|exception/i.test(this.line || '')) {
            level = 'error'
          }
          self.logger.log(String((logObject.level || logObject.lvl || logObject.severity || level)), logObject.message, logObject)
          self.debugLog('Stored Log: ' + JSON.stringify(logObject))
        }
{'fraction_non_alphanumeric': 0.1403613025909199, 'fraction_numerical': 0.0030900879486570002, 'mean_word_length': 2.9377632194665417, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4399268', 'n_tokens_mistral': 2805, 'n_tokens_neox': 2764, 'n_words': 643}