<commit_before># s2.2.7.4

codepageByLcid = require('../collation').codepageByLcid
iconvByLcid = require('../collation').iconvByLcid

TYPE = require('./data-type').TYPE
sprintf = require('sprintf').sprintf

parser = (buffer) ->
  columnCount = buffer.readUInt16LE()

  columns = []
  for c in [1..columnCount]
    userType = buffer.readUInt32LE()
    flags = buffer.readUInt16LE()
    typeNumber = buffer.readUInt8()
    type = TYPE[typeNumber]

    if !type
      throw new Error(sprintf('Unrecognised data type 0x%02X at offset 0x%04X', typeNumber, (buffer.position - 1)))

    #console.log(type)

    if (type.id & 0x30) == 0x20
      # xx10xxxx - s2.2.4.2.1.3
      # Variable length
      switch type.dataLengthLength
        when 1
          dataLength = buffer.readUInt8()
        when 2
          dataLength = buffer.readUInt16LE()
        when 4
          dataLength = buffer.readUInt32LE()
        else
          throw new Error("Unsupported dataLengthLength #{type.dataLengthLength} for data type #{type.name}")
    else
      dataLength = undefined

    if type.hasPrecision
      precision = buffer.readUInt8()
    else
      precision = undefined

    if type.hasScale
      scale = buffer.readUInt8()
    else
      scale = undefined

    if type.hasCollation
      # s2.2.5.1.2
      collationData = buffer.readBuffer(5)
      collation = {}

      collation.lcid = (collationData[2] & 0x0F) << 16
      collation.lcid |= collationData[1] << 8
      collation.lcid |= collationData[0]
      
      collation.codepage = codepageByLcid[collation.lcid]
      collation.iconv = iconvByLcid[collation.lcid]

      # This may not be extracting the correct nibbles in the correct order.
      collation.flags = collationData[3] >> 4
      collation.flags |= collationData[2] & 0xF0

      # This may not be extracting the correct nibble.
      collation.version = collationData[3] & 0x0F

      collation.sortId = collationData[4]
    else
      collation = undefined

    if type.hasTableName
      numberOfTableNameParts = buffer.readUInt8()
      tableName = for part in [1..numberOfTableNameParts]
        buffer.readUsVarchar('ucs2')
    else<commit_msg>Detect attempted use of an unconfigured LCID, and throw an error.<commit_after># s2.2.7.4

codepageByLcid = require('../collation').codepageByLcid
iconvByLcid = require('../collation').iconvByLcid

TYPE = require('./data-type').TYPE
sprintf = require('sprintf').sprintf

parser = (buffer) ->
  columnCount = buffer.readUInt16LE()

  columns = []
  for c in [1..columnCount]
    userType = buffer.readUInt32LE()
    flags = buffer.readUInt16LE()
    typeNumber = buffer.readUInt8()
    type = TYPE[typeNumber]

    if !type
      throw new Error(sprintf('Unrecognised data type 0x%02X at offset 0x%04X', typeNumber, (buffer.position - 1)))

    #console.log(type)

    if (type.id & 0x30) == 0x20
      # xx10xxxx - s2.2.4.2.1.3
      # Variable length
      switch type.dataLengthLength
        when 1
          dataLength = buffer.readUInt8()
        when 2
          dataLength = buffer.readUInt16LE()
        when 4
          dataLength = buffer.readUInt32LE()
        else
          throw new Error("Unsupported dataLengthLength #{type.dataLengthLength} for data type #{type.name}")
    else
      dataLength = undefined

    if type.hasPrecision
      precision = buffer.readUInt8()
    else
      precision = undefined

    if type.hasScale
      scale = buffer.readUInt8()
    else
      scale = undefined

    if type.hasCollation
      # s2.2.5.1.2
      collationData = buffer.readBuffer(5)
      collation = {}

      collation.lcid = (collationData[2] & 0x0F) << 16
      collation.lcid |= collationData[1] << 8
      collation.lcid |= collationData[0]
      
      collation.codepage = codepageByLcid[collation.lcid]
      collation.iconv = iconvByLcid[collation.lcid]
      if !collation.iconv
        throw new Error("No codepage for LCID #{collation.lcid}")

      # This may not be extracting the correct nibbles in the correct order.
      collation.flags = collationData[3] >> 4
      collation.flags |= collationData[2] & 0xF0

      # This may not be extracting the correct nibble.
      collation.version = collationData[3] & 0x0F

      collation.sortId = collationData[4]
    else
      collation = undefined

    if type.hasTableName
      numberOfTableNameParts = buffer.readUInt8()
      tableName = for part in [1..numberOfTableNameParts]
        buffer.readUsVarchar('ucs2')
    else
{'fraction_non_alphanumeric': 0.10401785714285715, 'fraction_numerical': 0.03080357142857143, 'mean_word_length': 2.6224737267582863, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7137957', 'n_tokens_mistral': 1496, 'n_tokens_neox': 1436, 'n_words': 328}