<commit_before>parser = require('../../../lib/token/row-token-parser')
TYPE = require('../../../lib/token/token').TYPE
dataTypeByName = require('../../../lib/token/data-type').typeByName

module.exports.int = (test) ->
  colMetaData = [
    type:
      name: 'Int'
      dataLength: 4
  ]

  value = 3

  buffer = new Buffer(1 + 4)
  pos = 0;

  buffer.writeUInt8(TYPE.ROW, pos); pos++
  buffer.writeUInt32LE(value, pos); pos += 4
  #console.log(buffer)

  token = parser(buffer, 1, colMetaData)
  #console.log(token)

  test.strictEqual(token.length, buffer.length - 1)
  test.strictEqual(token.columns.length, 1)
  test.ok(!token.columns[0].isNull)
  test.strictEqual(token.columns[0].value, value)
  test.strictEqual(token.columns[0].metadata, colMetaData[0])

  test.done()

module.exports.varChar = (test) ->
  colMetaData = [
    type:
      name: 'VarChar'
      dataLengthLength: 2
  ]

  value = 'abc'

  buffer = new Buffer(1 + 2 + 3)
  pos = 0;

  buffer.writeUInt8(TYPE.ROW, pos); pos++
  buffer.writeUInt16LE(value.length, pos); pos += colMetaData[0].type.dataLengthLength
  buffer.write(value, pos, 'ascii'); pos += value.length
  #console.log(buffer)

  token = parser(buffer, 1, colMetaData)
  #console.log(token)

  test.strictEqual(token.length, buffer.length - 1)
  test.strictEqual(token.columns.length, 1)
  test.ok(!token.columns[0].isNull)
  test.strictEqual(token.columns[0].value, value)
  test.strictEqual(token.columns[0].metadata, colMetaData[0])

  test.done()

module.exports.nVarChar = (test) ->
  colMetaData = [
    type:
      name: 'NVarChar'
      dataLengthLength: 2
  ]

  value = 'abc'

  buffer = new Buffer(1 + 2 + 6)
  pos = 0;

  buffer.writeUInt8(TYPE.ROW, pos); pos++
  buffer.writeUInt16LE(value.length * 2, pos); pos += colMetaData[0].type.dataLengthLength
  buffer.write(value, pos, 'ucs-2'); pos += (value.length * 2)
  #console.log(buffer)

  token = parser(buffer, 1, colMetaData)
  #console.log(token)

  test.strictEqual(token.length, buffer.length - 1)
  test.strictEqual(token.columns.length, 1)
  test.ok(!token.columns[0].isNull)
  test.strictEqual(token.columns[0].value, value)
  test.strictEqual(token.columns[0].metadata, colMetaData[0])

  test.done()

module.exports.intN = (test) ->
  colMetaData = [
    type:
      name: 'IntN'
      dataLengthLength: 1
  ]

  value = 0

  buffer = new Buffer(1 + 1)
  pos = 0;

  buffer.writeUInt8(TYPE.ROW, pos); pos++
  buffer.writeUInt8(value, pos); pos++
  #console.log(buffer)

  token = parser(buffer, 1, colMetaData)
  #console.log(token)

  test.strictEqual(token.length, buffer.length - 1)
  test.strictEqual(token.columns.length, 1)
  test.ok(!token.columns[0].value)
  test.ok(token.columns[0].isNull)

  test.done()<commit_msg>Simplify, and make more readable, row token parsing testing by using WritableTrackingBuffer.<commit_after>parser = require('../../../lib/token/row-token-parser')
TYPE = require('../../../lib/token/token').TYPE
dataTypeByName = require('../../../lib/token/data-type').typeByName
WritableBuffer = require('../../../lib/tracking-buffer/tracking-buffer').WritableTrackingBuffer

module.exports.int = (test) ->
  colMetaData = [type: dataTypeByName.Int]

  value = 3

  buffer = new WritableBuffer(0)
  buffer.writeUInt8(TYPE.ROW)
  buffer.writeUInt32LE(value)
  buffer = buffer.data
  #console.log(buffer)

  token = parser(buffer, 1, colMetaData)
  #console.log(token)

  test.strictEqual(token.length, buffer.length - 1)
  test.strictEqual(token.columns.length, 1)
  test.ok(!token.columns[0].isNull)
  test.strictEqual(token.columns[0].value, value)
  test.strictEqual(token.columns[0].metadata, colMetaData[0])

  test.done()

module.exports.varChar = (test) ->
  colMetaData = [type: dataTypeByName.VarChar]

  value = 'abc'

  buffer = new WritableBuffer(0, 'ascii')
  buffer.writeUInt8(TYPE.ROW)
  buffer.writeUInt16LE(value.length)
  buffer.writeString(value)
  buffer = buffer.data
  #console.log(buffer)

  token = parser(buffer, 1, colMetaData)
  #console.log(token)

  test.strictEqual(token.length, buffer.length - 1)
  test.strictEqual(token.columns.length, 1)
  test.ok(!token.columns[0].isNull)
  test.strictEqual(token.columns[0].value, value)
  test.strictEqual(token.columns[0].metadata, colMetaData[0])

  test.done()

module.exports.nVarChar = (test) ->
  colMetaData = [type: dataTypeByName.NVarChar]

  value = 'abc'

  buffer = new WritableBuffer(0, 'ucs2')
  buffer.writeUInt8(TYPE.ROW)
  buffer.writeUInt16LE(value.length * 2)
  buffer.writeString(value)
  buffer = buffer.data
  #console.log(buffer)

  token = parser(buffer, 1, colMetaData)
  #console.log(token)

  test.strictEqual(token.length, buffer.length - 1)
  test.strictEqual(token.columns.length, 1)
  test.ok(!token.columns[0].isNull)
  test.strictEqual(token.columns[0].value, value)
  test.strictEqual(token.columns[0].metadata, colMetaData[0])

  test.done()

module.exports.intN = (test) ->
  colMetaData = [type: dataTypeByName.IntN]

  value = 0

  buffer = new WritableBuffer(0)

  buffer.writeUInt8(TYPE.ROW)
  buffer.writeUInt8(value)
  buffer = buffer.data
  #console.log(buffer)

  token = parser(buffer, 1, colMetaData)
  #console.log(token)

  test.strictEqual(token.length, buffer.length - 1)
  test.strictEqual(token.columns.length, 1)
  test.ok(!token.columns[0].value)
  test.ok(token.columns[0].isNull)

  test.done()
{'fraction_non_alphanumeric': 0.16082089552238807, 'fraction_numerical': 0.020149253731343283, 'mean_word_length': 3.918348623853211, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '968175', 'n_tokens_mistral': 1966, 'n_tokens_neox': 1892, 'n_words': 320}