<commit_before># s2.2.7.17

TYPE = require('./data-type').TYPE

parser = (buffer, position, columnsMetaData) ->
  startPosition = position

  columns = for columnMetaData in columnsMetaData
    type = columnMetaData.type
    switch type.name
      when 'Int'
        value = buffer.readUInt32LE(position)
        position += type.dataLength
      when 'VarChar', 'Char'
        dataLength = buffer.readUInt16LE(position)
        position += 2
        value = buffer.toString('ascii', position, position + dataLength)
        position += dataLength
      when 'NVarChar'
        dataLength = buffer.readUInt16LE(position)
        position += 2
        value = buffer.toString('ucs-2', position, position + (dataLength))
        position += dataLength
      else
        error = "Unrecognised column type #{type.name}"
        break

    column =
      value: value
      metadata: columnMetaData

  if error
    token =
      name: 'ROW'
      error: error
  else
    token =
      name: 'ROW'
      length: position - startPosition
      event: 'row'
      columns: columns

module.exports = parser
<commit_msg>Expose row values by column name.<commit_after># s2.2.7.17

TYPE = require('./data-type').TYPE

parser = (buffer, position, columnsMetaData) ->
  startPosition = position

  columns = for columnMetaData in columnsMetaData
    type = columnMetaData.type
    switch type.name
      when 'Int'
        value = buffer.readUInt32LE(position)
        position += type.dataLength
      when 'VarChar', 'Char'
        dataLength = buffer.readUInt16LE(position)
        position += 2
        value = buffer.toString('ascii', position, position + dataLength)
        position += dataLength
      when 'NVarChar'
        dataLength = buffer.readUInt16LE(position)
        position += 2
        value = buffer.toString('ucs-2', position, position + (dataLength))
        position += dataLength
      else
        error = "Unrecognised column type #{type.name}"
        break

    column =
      value: value
      metadata: columnMetaData

  columns.byName = ->
    byName = {}

    for column in columns
      byName[column.metadata.colName] = column

    byName

  if error
    token =
      name: 'ROW'
      error: error
  else
    token =
      name: 'ROW'
      length: position - startPosition
      event: 'row'
      columns: columns

module.exports = parser

{'fraction_non_alphanumeric': 0.09674693705111956, 'fraction_numerical': 0.011829319814110688, 'mean_word_length': 2.3446327683615817, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7200287', 'n_tokens_mistral': 672, 'n_tokens_neox': 658, 'n_words': 182}