<commit_before>exports.name = "creationix/websocket-codec"
exports.version = "0.1.0"

local digest = require('openssl').digest.digest
local base64 = require('openssl').base64
local random = require('openssl').random
local hexToBin = require('creationix/hex-bin').hexToBin

local function applyMask(data, mask)
  local bytes = {
    [0] = string.byte(mask, 1),
    [1] = string.byte(mask, 2),
    [2] = string.byte(mask, 3),
    [3] = string.byte(mask, 4)
  }
  local out = {}
  for i = 1, #data do
    out[i] = string.char(
      bit.bxor(string.byte(data, i), bytes[(i - 1) % 4])
    )
  end
  return table.concat(out)
end

function exports.decode(chunk)
  if #chunk < 2 then return end
  local second = string.byte(chunk, 2)
  local len = bit.band(second, 0x7f)
  local offset
  if len == 126 then
    if #chunk < 4 then return end
    len = bit.bor(
      bit.lshift(string.byte(chunk, 3), 8),
      string.byte(chunk, 4))
    offset = 4
  elseif len == 127 then
    if #chunk < 10 then return end
    len = bit.bor(
      bit.lshift(string.byte(chunk, 3), 56),
      bit.lshift(string.byte(chunk, 4), 48),
      bit.lshift(string.byte(chunk, 5), 40),
      bit.lshift(string.byte(chunk, 6), 32),
      bit.lshift(string.byte(chunk, 7), 24),
      bit.lshift(string.byte(chunk, 8), 16),
      bit.lshift(string.byte(chunk, 9), 8),
      string.byte(chunk, 10))
    offset = 10
  else
    offset = 2
  end
  local mask = bit.band(second, 0x80) > 0
  if mask then
    offset = offset + 4
  end
  if #chunk < offset + len - 1 then return end

  local first = string.byte(chunk, 1)
  local payload = string.sub(chunk, offset + 1, offset + len)
  if mask then
    payload = applyMask(payload, string.sub(chunk, offset - 3, offset))
  end
  local extra = string.sub(chunk, offset + len + 1)
  return {
    fin = bit.band(first, 0x80) > 0,
    rsv1 = bit.band(first, 0x40) > 0,
    rsv2 = bit.band(first, 0x20) > 0,
    rsv3 = bit.band(first, 0x10) > 0,
    opcode = bit.band(first, 0xf),
    mask = mask,
    len = len,
    payload = payload
  }, extra
end

function exports.encode(item)
  if type(item) == "string" then
    item = {
      opcode = 2,
      payload = item
    }
  end
  local payload = item.payload
  assert(type(payload) == "string", "payload must be string")
  local len = #payload
  local chars = {
    string.char(bit.bor(0x80, item.opcode or 2)),
    string.char(bit.bor(item.mask and 0x80 or 0,
      len < 0x10 and len or len < 0x10000 and 126 or 127))
  }<commit_msg>websocket-codec v0.1.1: Add fin and rsv to encoder<commit_after>exports.name = "creationix/websocket-codec"
exports.version = "0.1.1"

local digest = require('openssl').digest.digest
local base64 = require('openssl').base64
local random = require('openssl').random
local hexToBin = require('creationix/hex-bin').hexToBin

local function applyMask(data, mask)
  local bytes = {
    [0] = string.byte(mask, 1),
    [1] = string.byte(mask, 2),
    [2] = string.byte(mask, 3),
    [3] = string.byte(mask, 4)
  }
  local out = {}
  for i = 1, #data do
    out[i] = string.char(
      bit.bxor(string.byte(data, i), bytes[(i - 1) % 4])
    )
  end
  return table.concat(out)
end

function exports.decode(chunk)
  if #chunk < 2 then return end
  local second = string.byte(chunk, 2)
  local len = bit.band(second, 0x7f)
  local offset
  if len == 126 then
    if #chunk < 4 then return end
    len = bit.bor(
      bit.lshift(string.byte(chunk, 3), 8),
      string.byte(chunk, 4))
    offset = 4
  elseif len == 127 then
    if #chunk < 10 then return end
    len = bit.bor(
      bit.lshift(string.byte(chunk, 3), 56),
      bit.lshift(string.byte(chunk, 4), 48),
      bit.lshift(string.byte(chunk, 5), 40),
      bit.lshift(string.byte(chunk, 6), 32),
      bit.lshift(string.byte(chunk, 7), 24),
      bit.lshift(string.byte(chunk, 8), 16),
      bit.lshift(string.byte(chunk, 9), 8),
      string.byte(chunk, 10))
    offset = 10
  else
    offset = 2
  end
  local mask = bit.band(second, 0x80) > 0
  if mask then
    offset = offset + 4
  end
  if #chunk < offset + len - 1 then return end

  local first = string.byte(chunk, 1)
  local payload = string.sub(chunk, offset + 1, offset + len)
  if mask then
    payload = applyMask(payload, string.sub(chunk, offset - 3, offset))
  end
  local extra = string.sub(chunk, offset + len + 1)
  return {
    fin = bit.band(first, 0x80) > 0,
    rsv1 = bit.band(first, 0x40) > 0,
    rsv2 = bit.band(first, 0x20) > 0,
    rsv3 = bit.band(first, 0x10) > 0,
    opcode = bit.band(first, 0xf),
    mask = mask,
    len = len,
    payload = payload
  }, extra
end

function exports.encode(item)
  if type(item) == "string" then
    item = {
      opcode = 2,
      payload = item
    }
  end
  local payload = item.payload
  assert(type(payload) == "string", "payload must be string")
  local len = #payload
  if item.fin == nil then item.fin = true end
  if item.opcode == nil then item.opcode = 2 end
  local chars = {
    string.char(bit.bor(
      item.fin and 0x80 or 0,
      item.rsv1 and 0x40 or 0,
      item.rsv2 and 0x20 or 0,
      item.rsv3 and 0x10 or 0,
      item.opcode
    )),
    string.char(bit.bor(
      item.mask and 0x80 or 0,
      len < 0x10 and len or len < 0x10000 and 126 or 127
    ))
  }
{'fraction_non_alphanumeric': 0.14048799085017155, 'fraction_numerical': 0.04708349218452154, 'mean_word_length': 2.7212765957446807, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7475413', 'n_tokens_mistral': 2104, 'n_tokens_neox': 1890, 'n_words': 542}