<commit_before>--
--  Author: <NAME> <<EMAIL>>
--
--  Copyright (C) 2013-2014 <NAME> <<EMAIL>>
--
--  Licensed according to the included 'LICENCE' document
--
--  This file is part of lua-lzqm library.
--

local ffi     = require "ffi"
local IS_WINDOWS = (ffi.os:lower() == 'windows') or
                   (package.config:sub(1,1) == '\\')

local function orequire(...)
  local err = ""
  for _, name in ipairs{...} do
    local ok, mod = pcall(require, name)
    if ok then return mod, name end
    err = err .. "\n" .. mod
  end
  error(err)
end

local function oload(t)
  local err = ""
  for _, name in ipairs(t) do
    local ok, mod = pcall(ffi.load, name)
    if ok then return mod, name end
    err = err .. "\n" .. mod
  end
  error(err)
end

local bit     = orequire("bit32", "bit")

local zlibs if IS_WINDOWS then
  zlibs = {
    "zmq",  "libzmq",
    "zmq4", "libzmq4",
    "zmq3", "libzmq3",
  }
else
  zlibs = {
    "zmq",      "libzmq",
    "zmq.so.4", "libzmq.so.4",
    "zmq.so.3", "libzmq.so.3",
    "/usr/local/lib/libzmq.so",
    "/usr/local/lib/libzmq.so.4",
    "/usr/local/lib/libzmq.so.3",
  }
end

local ok, libzmq3 = pcall( oload, zlibs )
if not ok then
  if pcall( require, "lzmq" ) then -- jus to load libzmq3
    libzmq3 = oload( zlibs )
  else error(libzmq3) end
end

local aint_t          = ffi.typeof("int[1]")
local aint16_t        = ffi.typeof("int16_t[1]")
local auint16_t       = ffi.typeof("uint16_t[1]")
local aint32_t        = ffi.typeof("int32_t[1]")
local auint32_t       = ffi.typeof("uint32_t[1]")
local aint64_t        = ffi.typeof("int64_t[1]")
local auint64_t       = ffi.typeof("uint64_t[1]")
local asize_t         = ffi.typeof("size_t[1]")
local vla_char_t      = ffi.typeof("char[?]")
local pvoid_t         = ffi.typeof("void*")
local pchar_t         = ffi.typeof("char*")
local uintptr_t       = ffi.typeof("uintptr_t")
local NULL            = ffi.cast(pvoid_t, 0)
local int16_size      = ffi.sizeof("int16_t")
local int32_size      = ffi.sizeof("int32_t")
local ptr_size        = ffi.sizeof(pvoid_t)
local fd_t, afd_t
if IS_WINDOWS and ffi.arch == 'x64' then
  fd_t, afd_t = "uint64_t", auint64_t
else
  fd_t, afd_t = "int", aint_t
end
local fd_size         = ffi.sizeof(fd_t)

-- !Note! this allocator could return same buffer more then once.
-- So you can not use this function to allocate 2 different buffers.
local function create_tmp_allocator(array_size, array_type)
  assert(type(array_size) == "number")
  assert(array_size > 0)

  if type(array_type) == 'string' then
    array_type = ffi.typeof(array_type)
  else
    array_type = array_type or vla_char_t
  end

  local buffer

  return function(len)
    if len <= array_size then
      if not buffer then
        buffer = ffi.new(array_type, array_size)
      end
      return buffer
    end
    return ffi.new(array_type, len)
  end
end

local header = [[
  void zmq_version (int *major, int *minor, int *patch);
]]
ffi.cdef(header)

local _M = {}

-- zmq_version
do

function _M.zmq_version()
  local major, minor, patch = ffi.new(aint_t, 0), ffi.new(aint_t, 0), ffi.new(aint_t, 0)
  libzmq3.zmq_version(major, minor, patch)
  return major[0], minor[0], patch[0]
end

end

local ZMQ_VERSION_MAJOR, ZMQ_VERSION_MINOR, ZMQ_VERSION_PATCH = _M.zmq_version()
assert(
  ((ZMQ_VERSION_MAJOR == 3) and (ZMQ_VERSION_MINOR >= 2)) or (ZMQ_VERSION_MAJOR == 4),
  "Unsupported ZMQ version: " .. ZMQ_VERSION_MAJOR .. "." .. ZMQ_VERSION_MINOR .. "." .. ZMQ_VERSION_PATCH
)

-- >=
local is_zmq_ge = function (major, minor, patch)
  if ZMQ_VERSION_MAJOR < major then return false end
  if ZMQ_VERSION_MAJOR > major then return true  end
  if ZMQ_VERSION_MINOR < minor then return false end
  if ZMQ_VERSION_MINOR > minor then return true  end
  if ZMQ_VERSION_PATCH < patch then return false  end
  return true
end

if is_zmq_ge(4, 2, 0) then
  header = [[
    typedef struct zmq_msg_t {unsigned char _ [64];} zmq_msg_t;
  ]]
  ffi.cdef(header)<commit_msg>Fix. message size changes in zmq 4.1.1<commit_after>--
--  Author: <NAME> <<EMAIL>>
--
--  Copyright (C) 2013-2014 <NAME> <<EMAIL>>
--
--  Licensed according to the included 'LICENCE' document
--
--  This file is part of lua-lzqm library.
--

local ffi     = require "ffi"
local IS_WINDOWS = (ffi.os:lower() == 'windows') or
                   (package.config:sub(1,1) == '\\')

local function orequire(...)
  local err = ""
  for _, name in ipairs{...} do
    local ok, mod = pcall(require, name)
    if ok then return mod, name end
    err = err .. "\n" .. mod
  end
  error(err)
end

local function oload(t)
  local err = ""
  for _, name in ipairs(t) do
    local ok, mod = pcall(ffi.load, name)
    if ok then return mod, name end
    err = err .. "\n" .. mod
  end
  error(err)
end

local bit     = orequire("bit32", "bit")

local zlibs if IS_WINDOWS then
  zlibs = {
    "zmq",  "libzmq",
    "zmq4", "libzmq4",
    "zmq3", "libzmq3",
  }
else
  zlibs = {
    "zmq",      "libzmq",
    "zmq.so.4", "libzmq.so.4",
    "zmq.so.3", "libzmq.so.3",
    "/usr/local/lib/libzmq.so",
    "/usr/local/lib/libzmq.so.4",
    "/usr/local/lib/libzmq.so.3",
  }
end

local ok, libzmq3 = pcall( oload, zlibs )
if not ok then
  if pcall( require, "lzmq" ) then -- jus to load libzmq3
    libzmq3 = oload( zlibs )
  else error(libzmq3) end
end

local aint_t          = ffi.typeof("int[1]")
local aint16_t        = ffi.typeof("int16_t[1]")
local auint16_t       = ffi.typeof("uint16_t[1]")
local aint32_t        = ffi.typeof("int32_t[1]")
local auint32_t       = ffi.typeof("uint32_t[1]")
local aint64_t        = ffi.typeof("int64_t[1]")
local auint64_t       = ffi.typeof("uint64_t[1]")
local asize_t         = ffi.typeof("size_t[1]")
local vla_char_t      = ffi.typeof("char[?]")
local pvoid_t         = ffi.typeof("void*")
local pchar_t         = ffi.typeof("char*")
local uintptr_t       = ffi.typeof("uintptr_t")
local NULL            = ffi.cast(pvoid_t, 0)
local int16_size      = ffi.sizeof("int16_t")
local int32_size      = ffi.sizeof("int32_t")
local ptr_size        = ffi.sizeof(pvoid_t)
local fd_t, afd_t
if IS_WINDOWS and ffi.arch == 'x64' then
  fd_t, afd_t = "uint64_t", auint64_t
else
  fd_t, afd_t = "int", aint_t
end
local fd_size         = ffi.sizeof(fd_t)

-- !Note! this allocator could return same buffer more then once.
-- So you can not use this function to allocate 2 different buffers.
local function create_tmp_allocator(array_size, array_type)
  assert(type(array_size) == "number")
  assert(array_size > 0)

  if type(array_type) == 'string' then
    array_type = ffi.typeof(array_type)
  else
    array_type = array_type or vla_char_t
  end

  local buffer

  return function(len)
    if len <= array_size then
      if not buffer then
        buffer = ffi.new(array_type, array_size)
      end
      return buffer
    end
    return ffi.new(array_type, len)
  end
end

local header = [[
  void zmq_version (int *major, int *minor, int *patch);
]]
ffi.cdef(header)

local _M = {}

-- zmq_version
do

function _M.zmq_version()
  local major, minor, patch = ffi.new(aint_t, 0), ffi.new(aint_t, 0), ffi.new(aint_t, 0)
  libzmq3.zmq_version(major, minor, patch)
  return major[0], minor[0], patch[0]
end

end

local ZMQ_VERSION_MAJOR, ZMQ_VERSION_MINOR, ZMQ_VERSION_PATCH = _M.zmq_version()
assert(
  ((ZMQ_VERSION_MAJOR == 3) and (ZMQ_VERSION_MINOR >= 2)) or (ZMQ_VERSION_MAJOR == 4),
  "Unsupported ZMQ version: " .. ZMQ_VERSION_MAJOR .. "." .. ZMQ_VERSION_MINOR .. "." .. ZMQ_VERSION_PATCH
)

-- >=
local is_zmq_ge = function (major, minor, patch)
  if ZMQ_VERSION_MAJOR < major then return false end
  if ZMQ_VERSION_MAJOR > major then return true  end
  if ZMQ_VERSION_MINOR < minor then return false end
  if ZMQ_VERSION_MINOR > minor then return true  end
  if ZMQ_VERSION_PATCH < patch then return false  end
  return true
end

if is_zmq_ge(4, 1, 1) then
  header = [[
    typedef struct zmq_msg_t {unsigned char _ [64];} zmq_msg_t;
  ]]
  ffi.cdef(header)
{'fraction_non_alphanumeric': 0.1345014496407412, 'fraction_numerical': 0.02306819614269507, 'mean_word_length': 3.003027245206862, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2194010', 'n_tokens_mistral': 3272, 'n_tokens_neox': 3135, 'n_words': 797}