<commit_before>﻿require 'ffi'
require_relative 'sodium/errors'
require_relative 'sodium/utils'
require_relative 'sodium/buffer'

module Sodium
  extend FFI::Library
  extend Utils
  ffi_lib :libsodium

  attach_function :init,  :sodium_init, [], :int

  attach_function :memcmp,  :sodium_memcmp,   [:buffer_in, :buffer_in, :size_t],  :int
  attach_function :memzero, :sodium_memzero,  [:pointer, :size_t],  :void
  attach_function :free,    :sodium_free,     [:pointer],           :void
  attach_function :sodium_mlock,              [:pointer, :size_t],  :int
  attach_function :sodium_munlock,            [:pointer, :size_t],  :int
  attach_function :sodium_malloc,             [:size_t],            :pointer
  attach_function :sodium_allocarray,         [:size_t, :size_t],   :pointer

  attach_function :sodium_bin2hex,  [:buffer_out, :size_t, :buffer_in, :size_t],  :string
  attach_function :sodium_hex2bin,  [:buffer_out, :size_t, :buffer_in, :size_t, :string, :pointer, :pointer],  :int

  module_function

  def mlock(addr, len)
    if sodium_mlock(addr, len) == 0
      true
    else
      raise MemoryError, "Could not lock length=#{len} bytes memory at address=#{addr.address}", caller
    end
  end

  def munlock(addr, len)
    if sodium_munlock(addr, len) == 0
      true
    else
      raise MemoryError, "Could not unlock length=#{len} bytes memory at address=#{addr.address}", caller
    end
  end

  def malloc(size)
    unless (mem = sodium_malloc(size)).null?
      mem
    else
      raise NoMemoryError, "Failed to allocate memory size=#{size} bytes", caller
    end
  end

  def allocarray(count, size)
    unless (mem = sodium_allocarray(count, size)).null?
      mem
    else
      raise NoMemoryError, "Failed to allocate memory size=#{count * size} bytes", caller
    end
  end

  def bin2hex(bin)
    bin_len = get_size(bin)
    hex = FFI::MemoryPointer.new(:char, bin_len * 2 + 1)
    sodium_bin2hex(hex, hex.size, bin, bin_len)
  end

  def hex2bin(hex, bin_maxlen, ignore = nil)
    bin = Sodium::Buffer.new(:uchar, bin_maxlen)
    if sodium_hex2bin(bin, bin_maxlen, hex, hex.bytesize, ignore, nil, nil) == 0
      bin
    else
      raise LengthError, "bin_maxlen=#{bin_maxlen} is too short", caller
    end
  end<commit_msg>return actual size of hex2bin conversion<commit_after>﻿require 'ffi'
require_relative 'sodium/errors'
require_relative 'sodium/utils'
require_relative 'sodium/buffer'

module Sodium
  extend FFI::Library
  extend Utils
  ffi_lib :libsodium

  attach_function :init,  :sodium_init, [], :int

  attach_function :memcmp,  :sodium_memcmp,   [:buffer_in, :buffer_in, :size_t],  :int
  attach_function :memzero, :sodium_memzero,  [:pointer, :size_t],  :void
  attach_function :free,    :sodium_free,     [:pointer],           :void
  attach_function :sodium_mlock,              [:pointer, :size_t],  :int
  attach_function :sodium_munlock,            [:pointer, :size_t],  :int
  attach_function :sodium_malloc,             [:size_t],            :pointer
  attach_function :sodium_allocarray,         [:size_t, :size_t],   :pointer

  attach_function :sodium_bin2hex,  [:buffer_out, :size_t, :buffer_in, :size_t],  :string
  attach_function :sodium_hex2bin,  [:buffer_out, :size_t, :buffer_in, :size_t, :string, :pointer, :pointer],  :int

  module_function

  def mlock(addr, len)
    if sodium_mlock(addr, len) == 0
      true
    else
      raise MemoryError, "Could not lock length=#{len} bytes memory at address=#{addr.address}", caller
    end
  end

  def munlock(addr, len)
    if sodium_munlock(addr, len) == 0
      true
    else
      raise MemoryError, "Could not unlock length=#{len} bytes memory at address=#{addr.address}", caller
    end
  end

  def malloc(size)
    unless (mem = sodium_malloc(size)).null?
      mem
    else
      raise NoMemoryError, "Failed to allocate memory size=#{size} bytes", caller
    end
  end

  def allocarray(count, size)
    unless (mem = sodium_allocarray(count, size)).null?
      mem
    else
      raise NoMemoryError, "Failed to allocate memory size=#{count * size} bytes", caller
    end
  end

  def bin2hex(bin)
    bin_len = get_size(bin)
    hex = FFI::MemoryPointer.new(:char, bin_len * 2 + 1)
    sodium_bin2hex(hex, hex.size, bin, bin_len)
  end

  def hex2bin(hex, bin_maxlen, ignore = nil)
    bin = Sodium::Buffer.new(:uchar, bin_maxlen)
    bin_len = FFI::MemoryPointer.new(:size_t)
    if sodium_hex2bin(bin, bin_maxlen, hex, hex.bytesize, ignore, bin_len, nil) == 0
      size = bin_len.size == 8 ? bin_len.read_ulong : bin_len.read_uint
      [bin, size]
    else
      raise LengthError, "bin_maxlen=#{bin_maxlen} is too short", caller
    end
  end
{'fraction_non_alphanumeric': 0.11389424106187113, 'fraction_numerical': 0.005138086062941554, 'mean_word_length': 3.445290199809705, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2702234', 'n_tokens_mistral': 1681, 'n_tokens_neox': 1585, 'n_words': 412}