<commit_before>--
--  Author: <NAME> <<EMAIL>>
--
--  Copyright (C) 2014 <NAME> <<EMAIL>>
--
--  Licensed according to the included 'LICENSE' document
--
--  This file is part of Lyre library.
--

local bit    = require "bit32"
local struct = require "struct"

local unpack = unpack or table.unpack

local function count(t)
  local n = 0
  for _ in pairs(t) do n = n + 1 end
  return n
end

---------------------------------------------------------------------
local Iter = {} do
Iter.__index = Iter

function Iter:new(data)
  return setmetatable({_data = data, _pos = 1, _len = #data}, self)
end<commit_msg>Change. Allow use `bit` library (e.g. on LuaJIT)<commit_after>--
--  Author: <NAME> <<EMAIL>>
--
--  Copyright (C) 2014 <NAME> <<EMAIL>>
--
--  Licensed according to the included 'LICENSE' document
--
--  This file is part of Lyre library.
--

local bit repeat
  local ok
  ok, bit = pcall(require, "bit32")
  if ok then break end
  bit = require "bit"
until true

local struct = require "struct"

local unpack = unpack or table.unpack

local function count(t)
  local n = 0
  for _ in pairs(t) do n = n + 1 end
  return n
end

---------------------------------------------------------------------
local Iter = {} do
Iter.__index = Iter

function Iter:new(data)
  return setmetatable({_data = data, _pos = 1, _len = #data}, self)
end
{'fraction_non_alphanumeric': 0.2222222222222222, 'fraction_numerical': 0.013422818791946308, 'mean_word_length': 2.0993071593533488, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2815034', 'n_tokens_mistral': 441, 'n_tokens_neox': 421, 'n_words': 159}