<commit_before>require 'rubygems'

require 'ffi_yajl/encoder'
require 'ffi_yajl/parser'
require 'ffi'
require 'libyajl2'
begin
  require 'fiddle'
rescue LoadError
  require 'dl'
end

module FFI_Yajl
  class Parser
    # FIXME: DRY with ffi_yajl/ffi.rb
    libname = ::FFI.map_library_name("yajl")
    libpath = File.expand_path(File.join(Libyajl2.opt_path, libname))
    libpath.gsub!(/dylib/, 'bundle')
    if defined?(Fiddle) && Fiddle.respond_to?(:dlopen)
      ::Fiddle.dlopen(libpath)
    else
      ::DL.dlopen(libpath)
    end
    require 'ffi_yajl/ext/parser'
    include FFI_Yajl::Ext::Parser
  end

  class Encoder
    # FIXME: DRY with ffi_yajl/ffi.rb
    libname = ::FFI.map_library_name("yajl")
    libpath = File.expand_path(File.join(Libyajl2.opt_path, libname))
    libpath.gsub!(/dylib/, 'bundle')
    if defined?(Fiddle) && Fiddle.respond_to?(:dlopen)
      ::Fiddle.dlopen(libpath)
    else
      ::DL.dlopen(libpath)
    end
    require 'ffi_yajl/ext/encoder'
    include FFI_Yajl::Ext::Encoder
  end<commit_msg>working on dlopen problem<commit_after>require 'rubygems'

require 'ffi_yajl/encoder'
require 'ffi_yajl/parser'
require 'ffi'
require 'libyajl2'
begin
  require 'fiddle'
rescue LoadError
end
begin
  require 'dl'
rescue LoadError
end

module FFI_Yajl
  # FIXME: DRY with ffi_yajl/ffi.rb
  libname = ::FFI.map_library_name("yajl")
  libpath = File.expand_path(File.join(Libyajl2.opt_path, libname))
  libpath.gsub!(/dylib/, 'bundle')
  libpath = ::FFI.map_library_name("yajl") unless File.exist?(libpath)
  if defined?(Fiddle) && Fiddle.respond_to?(:dlopen)
    ::Fiddle.dlopen(libpath)
  elsif defined?(DL) && DL.respond_to?(:dlopen)
    ::DL.dlopen(libpath)
  else
    ffi_lib libpath
  end

  class Parser
    require 'ffi_yajl/ext/parser'
    include FFI_Yajl::Ext::Parser
  end

  class Encoder
    require 'ffi_yajl/ext/encoder'
    include FFI_Yajl::Ext::Encoder
  end
{'fraction_non_alphanumeric': 0.12703412073490813, 'fraction_numerical': 0.0026246719160104987, 'mean_word_length': 4.082666666666666, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4641578', 'n_tokens_mistral': 778, 'n_tokens_neox': 780, 'n_words': 137}