<commit_before>require 'ffi'

module PyCall
  class PyObject < FFI::Struct
    layout ob_refcnt: :ssize_t,
           ob_type:   :pointer
  end

  class PyTypeObject < FFI::Struct
    layout ob_base: PyObject,
           ob_size: :ssize_t,
           tp_name: :string
  end

  module LibPython
    extend FFI::Library

    private_class_method

    def self.find_libpython(python = nil)
      python ||= 'python'
      python_config = investigate_python_config(python)

      v = python_config[:VERSION]
      libprefix = FFI::Platform::LIBPREFIX
      libs = [ "#{libprefix}python#{v}", "#{libprefix}python" ]
      lib = python_config[:LIBRARY]
      libs.unshift(File.basename(lib, File.extname(lib))) if lib
      lib = python_config[:LDLIBRARY]
      libs.unshift(lib, File.basename(lib)) if lib
      libs.uniq!

      executable = python_config[:executable]
      libpaths = [ python_config[:LIBDIR] ]
      if FFI::Platform.windows?
        libpaths << dirname(executable)
      else
        libpaths << File.expand_path('../../lib', executable)
      end
      libpaths << python_config[:PYTHONFRAMEWORKPREFIX] if FFI::Platform.mac?

      exec_prefix = python_config[:exec_prefix]
      libpaths << exec_prefix << File.join(exec_prefix, 'lib')

      unless ENV['PYTHONHOME']
        # PYTHONHOME tells python where to look for both pure python and binary modules.
        # When it is set, it replaces both `prefix` and `exec_prefix`
        # and we thus need to set it to both in case they differ.
        # This is also what the documentation recommends.
        # However, they are documented to always be the same on Windows,
        # where it causes problems if we try to include both.
        if FFI::Platform.windows?
          ENV['PYTHONHOME'] = exec_prefix
        else
          ENV['PYTHONHOME'] = [python_config[:prefix], exec_prefix].join(':')
        end

        # Unfortunately, setting PYTHONHOME screws up Canopy's Python distribution?
        unless system(python, '-c', 'import site', out: File::NULL, err: File::NULL)
          ENV['PYTHONHOME'] = nil
        end
      end

      # Find libpython (we hope):
      libsuffix = FFI::Platform::LIBSUFFIX
      libs.each do |lib|
        libpaths.each do |libpath|
          libpath_lib = File.join(libpath, lib)
          if File.file?("#{libpath_lib}.#{libsuffix}")
            libs = ffi_lib("#{libpath_lib}.#{libsuffix}")
            return libs.first
          end
        end
      end
    end

    def self.investigate_python_config(python)
      python_env = { 'PYTHONIOENCODING' => 'UTF-8' }
      IO.popen(python_env, [python, python_investigator_py], 'r') do |io|
        {}.tap do |config|
          io.each_line do |line|
            key, value = line.chomp.split(': ', 2)
            config[key.to_sym] = value
          end
        end
      end
    end

    def self.python_investigator_py
      File.expand_path('../python/investigator.py', __FILE__)
    end

    ffi_lib_flags :lazy, :global
    libpython = find_libpython ENV['PYTHON']

    # --- global variables ---

    attach_variable :_Py_NoneStruct, PyObject

    def self.Py_None
      _Py_NoneStruct
    end

    if libpython.find_variable('PyInt_Type')
      has_PyInt_Type = true
      attach_variable :PyInt_Type, PyTypeObject
    else
      has_PyInt_Type = false
      attach_variable :PyInt_Type, :PyLong_Type, PyTypeObject
    end

    attach_variable :PyLong_Type, PyTypeObject
    attach_variable :PyBool_Type, PyTypeObject
    attach_variable :PyFloat_Type, PyTypeObject
    attach_variable :PyComplex_Type, PyTypeObject
    attach_variable :PyUnicode_Type, PyTypeObject

    if libpython.find_symbol('PyString_FromStringAndSize')
      string_as_bytes = false
      attach_variable :PyString_Type, PyTypeObject
    else
      string_as_bytes = true
      attach_variable :PyString_Type, :PyBytes_Type, PyTypeObject
    end

    attach_variable :PyList_Type, PyTypeObject
    attach_variable :PyTuple_Type, PyTypeObject
    attach_variable :PyDict_Type, PyTypeObject
    attach_variable :PySet_Type, PyTypeObject

    # --- functions ---

    attach_function :Py_GetVersion, [], :string
    attach_function :Py_InitializeEx, [:int], :void
    attach_function :Py_IsInitialized, [], :int

    # Comparing two objects
    attach_function :PyObject_RichCompare, [PyObject.by_ref, PyObject.by_ref, :int], PyObject.by_ref

    # Accessing Object's attributes
    attach_function :PyObject_GetAttrString, [PyObject.by_ref, :string], PyObject.by_ref
    attach_function :PyObject_SetAttrString, [PyObject.by_ref, :string, PyObject.by_ref], :int
    attach_function :PyObject_HasAttrString, [PyObject.by_ref, :string], :int

    # Accessing Object's items
    attach_function :PyObject_GetItem, [PyObject.by_ref, PyObject.by_ref], PyObject.by_ref
    attach_function :PyObject_SetItem, [PyObject.by_ref, PyObject.by_ref, PyObject.by_ref], :int
    attach_function :PyObject_DelItem, [PyObject.by_ref, PyObject.by_ref], :int

    # Calling a object as a function
    attach_function :PyObject_Call, [PyObject.by_ref, PyObject.by_ref, PyObject.by_ref], PyObject.by_ref

    attach_function :PyObject_IsInstance, [PyObject.by_ref, PyTypeObject.by_ref], :int

    # Bool

    attach_function :PyBool_FromLong, [:long], PyObject.by_ref

    # Integer

    if has_PyInt_Type
      attach_function :PyInt_AsSsize_t, [PyObject.by_ref], :ssize_t<commit_msg>:hocho:<commit_after>require 'ffi'

module PyCall
  class PyObject < FFI::Struct
    layout ob_refcnt: :ssize_t,
           ob_type:   :pointer
  end

  class PyTypeObject < FFI::Struct
    layout ob_base: PyObject,
           ob_size: :ssize_t,
           tp_name: :string
  end

  module LibPython
    extend FFI::Library

    private_class_method

    def self.find_libpython(python = nil)
      python ||= 'python'
      python_config = investigate_python_config(python)

      v = python_config[:VERSION]
      libprefix = FFI::Platform::LIBPREFIX
      libs = [ "#{libprefix}python#{v}", "#{libprefix}python" ]
      lib = python_config[:LIBRARY]
      libs.unshift(File.basename(lib, File.extname(lib))) if lib
      lib = python_config[:LDLIBRARY]
      libs.unshift(lib, File.basename(lib)) if lib
      libs.uniq!

      executable = python_config[:executable]
      libpaths = [ python_config[:LIBDIR] ]
      if FFI::Platform.windows?
        libpaths << dirname(executable)
      else
        libpaths << File.expand_path('../../lib', executable)
      end
      libpaths << python_config[:PYTHONFRAMEWORKPREFIX] if FFI::Platform.mac?

      exec_prefix = python_config[:exec_prefix]
      libpaths << exec_prefix << File.join(exec_prefix, 'lib')

      unless ENV['PYTHONHOME']
        # PYTHONHOME tells python where to look for both pure python and binary modules.
        # When it is set, it replaces both `prefix` and `exec_prefix`
        # and we thus need to set it to both in case they differ.
        # This is also what the documentation recommends.
        # However, they are documented to always be the same on Windows,
        # where it causes problems if we try to include both.
        if FFI::Platform.windows?
          ENV['PYTHONHOME'] = exec_prefix
        else
          ENV['PYTHONHOME'] = [python_config[:prefix], exec_prefix].join(':')
        end

        # Unfortunately, setting PYTHONHOME screws up Canopy's Python distribution?
        unless system(python, '-c', 'import site', out: File::NULL, err: File::NULL)
          ENV['PYTHONHOME'] = nil
        end
      end

      # Find libpython (we hope):
      libsuffix = FFI::Platform::LIBSUFFIX
      libs.each do |lib|
        libpaths.each do |libpath|
          libpath_lib = File.join(libpath, lib)
          if File.file?("#{libpath_lib}.#{libsuffix}")
            libs = ffi_lib("#{libpath_lib}.#{libsuffix}")
            return libs.first
          end
        end
      end
    end

    def self.investigate_python_config(python)
      python_env = { 'PYTHONIOENCODING' => 'UTF-8' }
      IO.popen(python_env, [python, python_investigator_py], 'r') do |io|
        {}.tap do |config|
          io.each_line do |line|
            key, value = line.chomp.split(': ', 2)
            config[key.to_sym] = value
          end
        end
      end
    end

    def self.python_investigator_py
      File.expand_path('../python/investigator.py', __FILE__)
    end

    ffi_lib_flags :lazy, :global
    libpython = find_libpython ENV['PYTHON']

    # --- global variables ---

    attach_variable :_Py_NoneStruct, PyObject

    def self.Py_None
      _Py_NoneStruct
    end

    if libpython.find_variable('PyInt_Type')
      has_PyInt_Type = true
      attach_variable :PyInt_Type, PyTypeObject
    else
      has_PyInt_Type = false
      attach_variable :PyInt_Type, :PyLong_Type, PyTypeObject
    end

    attach_variable :PyLong_Type, PyTypeObject
    attach_variable :PyBool_Type, PyTypeObject
    attach_variable :PyFloat_Type, PyTypeObject
    attach_variable :PyComplex_Type, PyTypeObject
    attach_variable :PyUnicode_Type, PyTypeObject

    if libpython.find_symbol('PyString_FromStringAndSize')
      string_as_bytes = false
      attach_variable :PyString_Type, PyTypeObject
    else
      string_as_bytes = true
      attach_variable :PyString_Type, :PyBytes_Type, PyTypeObject
    end

    attach_variable :PyList_Type, PyTypeObject
    attach_variable :PyTuple_Type, PyTypeObject
    attach_variable :PyDict_Type, PyTypeObject
    attach_variable :PySet_Type, PyTypeObject

    # --- functions ---

    attach_function :Py_GetVersion, [], :string
    attach_function :Py_InitializeEx, [:int], :void
    attach_function :Py_IsInitialized, [], :int

    # Object

    attach_function :PyObject_RichCompare, [PyObject.by_ref, PyObject.by_ref, :int], PyObject.by_ref
    attach_function :PyObject_GetAttrString, [PyObject.by_ref, :string], PyObject.by_ref
    attach_function :PyObject_SetAttrString, [PyObject.by_ref, :string, PyObject.by_ref], :int
    attach_function :PyObject_HasAttrString, [PyObject.by_ref, :string], :int
    attach_function :PyObject_GetItem, [PyObject.by_ref, PyObject.by_ref], PyObject.by_ref
    attach_function :PyObject_SetItem, [PyObject.by_ref, PyObject.by_ref, PyObject.by_ref], :int
    attach_function :PyObject_DelItem, [PyObject.by_ref, PyObject.by_ref], :int
    attach_function :PyObject_Call, [PyObject.by_ref, PyObject.by_ref, PyObject.by_ref], PyObject.by_ref
    attach_function :PyObject_IsInstance, [PyObject.by_ref, PyTypeObject.by_ref], :int

    # Bool

    attach_function :PyBool_FromLong, [:long], PyObject.by_ref

    # Integer

    if has_PyInt_Type
      attach_function :PyInt_AsSsize_t, [PyObject.by_ref], :ssize_t
{'fraction_non_alphanumeric': 0.09738072965388213, 'fraction_numerical': 0.00037418147801683815, 'mean_word_length': 3.091465748182166, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5967383', 'n_tokens_mistral': 3391, 'n_tokens_neox': 3358, 'n_words': 767}