<commit_before># coding: utf-8
module Natto

  # Module `Binding` encapsulates methods and behavior 
  # which are made available via `FFI` bindings to 
  # `mecab`.
  module Binding
    require 'ffi'
    require 'rbconfig'
    extend FFI::Library

    # String name for the environment variable used by 
    # `Natto` to indicate the exact name / full path
    # to the `mecab` library.
    MECAB_PATH = 'MECAB_PATH'.freeze
    
    # @private
    def self.included(base)
      base.extend(ClassMethods)
    end

    # Returns the file path to the `mecab` library based on 
    # the runtime environment.
    # 
    # @return file path to the `mecab` library
    # @raise [LoadError] if the library cannot be located
    def self.find_library
      return File.realpath(ENV[MECAB_PATH]) if ENV[MECAB_PATH]

      host_os = RbConfig::CONFIG['host_os']

      if host_os =~ /mswin|mingw/i
        require 'win32/registry'
        begin
          base = nil
          Win32::Registry::HKEY_CURRENT_USER.open('Software\MeCab') do |r|
            base = r['mecabrc'].split('etc').first
          end
          lib = File.join(base, 'bin/libmecab.dll')
          File.realpath(lib)
        rescue
          raise LoadError, "Please set #{MECAB_PATH} to the full path to libmecab.dll"
        end
      else
        require 'open3'
        if host_os =~ /darwin/i
          ext = 'dylib'
        else
          ext = 'so'
        end

        begin
          base, lib = nil, nil
          cmd = 'mecab-config --libs'
          Open3.popen3(cmd) do |stdin,stdout,stderr|
            toks = stdout.read.split
            base = toks[0][2..-1]
            lib  = toks[1][2..-1]
          end
          File.realpath(File.join(base, "lib#{lib}.#{ext}"))
        rescue
          raise LoadError, "Please set #{MECAB_PATH} to the full path to libmecab.#{ext}"
        end
      end
    end

    ffi_lib find_library

    # new interface
    attach_function :mecab_model_new2, [:string], :pointer
    attach_function :mecab_model_destroy, [:pointer], :void
    attach_function :mecab_model_dictionary_info, [:pointer], :pointer

    # old interface
    attach_function :mecab_new2, [:string], :pointer
    attach_function :mecab_version, [], :string
    attach_function :mecab_strerror, [:pointer],:string
    attach_function :mecab_destroy, [:pointer], :void
    attach_function :mecab_set_partial, [:pointer, :int], :void 
    attach_function :mecab_set_theta, [:pointer, :float], :void
    attach_function :mecab_set_lattice_level, [:pointer, :int], :void <commit_msg>binding.rb edited online with Bitbucket; using File.absolute_path rather than realpath.<commit_after># coding: utf-8
module Natto

  # Module `Binding` encapsulates methods and behavior 
  # which are made available via `FFI` bindings to 
  # `mecab`.
  module Binding
    require 'ffi'
    require 'rbconfig'
    extend FFI::Library

    # String name for the environment variable used by 
    # `Natto` to indicate the exact name / full path
    # to the `mecab` library.
    MECAB_PATH = 'MECAB_PATH'.freeze
    
    # @private
    def self.included(base)
      base.extend(ClassMethods)
    end

    # Returns the absolute file path to the `mecab` library based on 
    # the runtime environment.
    # 
    # @return file path to the `mecab` library
    # @raise [LoadError] if the library cannot be located
    def self.find_library
      return File.absolute_path(ENV[MECAB_PATH]) if ENV[MECAB_PATH]

      host_os = RbConfig::CONFIG['host_os']

      if host_os =~ /mswin|mingw/i
        require 'win32/registry'
        begin
          base = nil
          Win32::Registry::HKEY_CURRENT_USER.open('Software\MeCab') do |r|
            base = r['mecabrc'].split('etc').first
          end
          lib = File.join(base, 'bin/libmecab.dll')
          File.absolute_path(lib)
        rescue
          raise LoadError, "Please set #{MECAB_PATH} to the full path to libmecab.dll"
        end
      else
        require 'open3'
        if host_os =~ /darwin/i
          ext = 'dylib'
        else
          ext = 'so'
        end

        begin
          base, lib = nil, nil
          cmd = 'mecab-config --libs'
          Open3.popen3(cmd) do |stdin,stdout,stderr|
            toks = stdout.read.split
            base = toks[0][2..-1]
            lib  = toks[1][2..-1]
          end
          File.absolute_path(File.join(base, "lib#{lib}.#{ext}"))
        rescue
          raise LoadError, "Please set #{MECAB_PATH} to the full path to libmecab.#{ext}"
        end
      end
    end

    ffi_lib find_library

    # new interface
    attach_function :mecab_model_new2, [:string], :pointer
    attach_function :mecab_model_destroy, [:pointer], :void
    attach_function :mecab_model_dictionary_info, [:pointer], :pointer

    # old interface
    attach_function :mecab_new2, [:string], :pointer
    attach_function :mecab_version, [], :string
    attach_function :mecab_strerror, [:pointer],:string
    attach_function :mecab_destroy, [:pointer], :void
    attach_function :mecab_set_partial, [:pointer, :int], :void 
    attach_function :mecab_set_theta, [:pointer, :float], :void
    attach_function :mecab_set_lattice_level, [:pointer, :int], :void 
{'fraction_non_alphanumeric': 0.10540436949022614, 'fraction_numerical': 0.006132617861249521, 'mean_word_length': 2.4608753315649867, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4852020', 'n_tokens_mistral': 1734, 'n_tokens_neox': 1691, 'n_words': 444}