<commit_before>var _ = require('underscore')
  , utils = require('./utils')
  , Segment = require('./segment')

module.exports = Message

function Message(segments) {
  if (!(this instanceof Message))
    return new Message(segments)
  if (segments && Array.isArray(segments)) {
    this.segments = segments
    this.segmentTypes = _.unique(
      _.pluck(this.segments, 'SegmentType')
    )
    var len = segments.length
    for (var i=0; i<len; i++) {
      if (utils.segmentIsHeader(segments[i])) {
        this.header = segments[i]
        break
      }
    }
  } else if (segments instanceof Segment) {
    this.segments = [segments]
    if (utils.segmentIsHeader(segments)) {
      this.header = segments
    }
    this.segmentTypes = [segments.parsed.SegmentType]
  } else {
    this.segments = []
    this.segmentTypes = []
    this.header = null
  }
}

Message.prototype.hasSegments = function() {
  return this.segments.length !== 0
}

Message.prototype.addSegment = function(segment) {
  this.segments.push(segment)
  var t = segment.parsed.SegmentType
  if (!(~this.segmentTypes.indexOf(t))) {
    this.segmentTypes.push(t)
  }
  if (utils.segmentTypeIsHeader(t)) {
    this.header = segment
  }
}

Message.prototype.getHeader = function() {
  return this.header
}

Message.prototype.delimiters = function() {
  if (!this.header) return {
    segment: '\r',
    field: '|',
    component: '^',
    subcomponent: '&',
    repetition: '~',
    escape: '\\'
  }

  var chars = this.header.parsed.EncodingCharacters
  return {
    segment: '\r',
    field: chars[0],
    component: chars[1],
    subcomponent: chars[2],
    repetition: chars[3],
    escape: chars[4]
  }
}
<commit_msg>Add defaults just in case the encoding characters are invalid<commit_after>var _ = require('underscore')
  , utils = require('./utils')
  , Segment = require('./segment')

module.exports = Message

function Message(segments) {
  if (!(this instanceof Message))
    return new Message(segments)
  if (segments && Array.isArray(segments)) {
    this.segments = segments
    this.segmentTypes = _.unique(
      _.pluck(this.segments, 'SegmentType')
    )
    var len = segments.length
    for (var i=0; i<len; i++) {
      if (utils.segmentIsHeader(segments[i])) {
        this.header = segments[i]
        break
      }
    }
  } else if (segments instanceof Segment) {
    this.segments = [segments]
    if (utils.segmentIsHeader(segments)) {
      this.header = segments
    }
    this.segmentTypes = [segments.parsed.SegmentType]
  } else {
    this.segments = []
    this.segmentTypes = []
    this.header = null
  }
}

Message.prototype.hasSegments = function() {
  return this.segments.length !== 0
}

Message.prototype.addSegment = function(segment) {
  this.segments.push(segment)
  var t = segment.parsed.SegmentType
  if (!(~this.segmentTypes.indexOf(t))) {
    this.segmentTypes.push(t)
  }
  if (utils.segmentTypeIsHeader(t)) {
    this.header = segment
  }
}

Message.prototype.getHeader = function() {
  return this.header
}

Message.prototype.delimiters = function() {
  if (!this.header) return {
    segment: '\r',
    field: '|',
    component: '^',
    subcomponent: '&',
    repetition: '~',
    escape: '\\'
  }

  var chars = this.header.parsed.EncodingCharacters
  return {
    segment: '\r',
    field: chars[0] || '|',
    component: chars[1] || '^',
    subcomponent: chars[2] || '&',
    repetition: chars[3] || '~',
    escape: chars[4] || '\\'
  }
}

{'fraction_non_alphanumeric': 0.15273775216138327, 'fraction_numerical': 0.004034582132564841, 'mean_word_length': 3.008083140877598, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4175483', 'n_tokens_mistral': 1158, 'n_tokens_neox': 1149, 'n_words': 223}