<commit_before>require 'ffi'

module LibC
    extend FFI::Library

    ffi_lib FFI::Library::LIBC

    # attatch free
    attach_function :free, [:pointer], :void
end

module Strcmp
    class << self
        extend FFI::Library

        lib_dir = File.expand_path(File.join(File.dirname(__FILE__), "../lib"))
        ffi_lib File.join(lib_dir, "libstrcmp.so")

        def damerau_levenshtein(str1, str2)
            is_string(str1)
            is_string(str2)
            ffi_damerau_levenshtein(str1, str2)
        end

        def hamming(str1, str2)
            is_string(str1)
            is_string(str2)
            ffi_hamming(str1, str2)
        end

        def jaro(str1, str2)
            is_string(str1)
            is_string(str2)
            ffi_jaro(str1, str2)
        end

        def jaro_winkler(str1, str2)
            is_string(str1)
            is_string(str2)
            ffi_jaro_winkler(str1, str2)
        end

        def levenshtein(str1, str2)
            is_string(str1)
            is_string(str2)
            ffi_levenshtein(str1, str2)
        end

        def optimal_string_alignment(str1, str2)
            is_string(str1)
            is_string(str2)
            ffi_optimal_string_alignment(str1, str2)
        end

        def sift3(str1, str2)
            is_string(str1)
            is_string(str2)
            ffi_sift3(str1, str2)
        end

        def sift3B(str1, str2)
            is_string(str1)
            is_string(str2)
            ffi_sift3B(str1, str2)
        end

        def soundex(str)
            is_string(str)
            ptr = ffi_soundex(str)
            result = String.new(ptr.read_string)
            LibC.free(ptr)

            result
        end

        def refined_soundex(str)
            is_string(str)
            ptr = ffi_refined_soundex(str)
            result = String.new(ptr.read_string)
            LibC.free(ptr)

            result
        end

        private

        # distance
        attach_function :ffi_damerau_levenshtein, :damerau_levenshtein, [:string, :string], :uint
        attach_function :ffi_hamming, :hamming, [:string, :string], :uint
        attach_function :ffi_jaro, :jaro, [:string, :string], :double
        attach_function :ffi_jaro_winkler, :jaro_winkler, [:string, :string], :double
        attach_function :ffi_levenshtein, :levenshtein, [:string, :string], :uint
        attach_function :ffi_optimal_string_alignment, :optimal_string_alignment, [:string, :string], :uint
        attach_function :ffi_sift3, :sift3, [:string, :string, :uint], :double
        attach_function :ffi_sift3B, :sift3B, [:string, :string, :uint, :double], :double

        # phonetic
        attach_function :ffi_soundex, :soundex, [:string], :pointer
        attach_function :ffi_refined_soundex, :refined_soundex, [:string], :pointer

        # helper to verify strings
        def is_string(object)
            unless object.kind_of? String
                raise TypeError,
                    "Wrong argument type #{object.class} (expected String)"
            end
        end
    end
end<commit_msg>Fixed binding for hamming distance, changing the return value to int instead of uint. Now it will return -1 if there was an error.<commit_after>require 'ffi'

module LibC
    extend FFI::Library

    ffi_lib FFI::Library::LIBC

    # attatch free
    attach_function :free, [:pointer], :void
end

module Strcmp
    class << self
        extend FFI::Library

        lib_dir = File.expand_path(File.join(File.dirname(__FILE__), "../lib"))
        ffi_lib File.join(lib_dir, "libstrcmp.so")

        def damerau_levenshtein(str1, str2)
            is_string(str1)
            is_string(str2)
            ffi_damerau_levenshtein(str1, str2)
        end

        def hamming(str1, str2)
            is_string(str1)
            is_string(str2)
            ffi_hamming(str1, str2)
        end

        def jaro(str1, str2)
            is_string(str1)
            is_string(str2)
            ffi_jaro(str1, str2)
        end

        def jaro_winkler(str1, str2)
            is_string(str1)
            is_string(str2)
            ffi_jaro_winkler(str1, str2)
        end

        def levenshtein(str1, str2)
            is_string(str1)
            is_string(str2)
            ffi_levenshtein(str1, str2)
        end

        def optimal_string_alignment(str1, str2)
            is_string(str1)
            is_string(str2)
            ffi_optimal_string_alignment(str1, str2)
        end

        def sift3(str1, str2)
            is_string(str1)
            is_string(str2)
            ffi_sift3(str1, str2)
        end

        def sift3B(str1, str2)
            is_string(str1)
            is_string(str2)
            ffi_sift3B(str1, str2)
        end

        def soundex(str)
            is_string(str)
            ptr = ffi_soundex(str)
            result = String.new(ptr.read_string)
            LibC.free(ptr)

            result
        end

        def refined_soundex(str)
            is_string(str)
            ptr = ffi_refined_soundex(str)
            result = String.new(ptr.read_string)
            LibC.free(ptr)

            result
        end

        private

        # distance
        attach_function :ffi_damerau_levenshtein, :damerau_levenshtein, [:string, :string], :uint
        attach_function :ffi_hamming, :hamming, [:string, :string], :int
        attach_function :ffi_jaro, :jaro, [:string, :string], :double
        attach_function :ffi_jaro_winkler, :jaro_winkler, [:string, :string], :double
        attach_function :ffi_levenshtein, :levenshtein, [:string, :string], :uint
        attach_function :ffi_optimal_string_alignment, :optimal_string_alignment, [:string, :string], :uint
        attach_function :ffi_sift3, :sift3, [:string, :string, :uint], :double
        attach_function :ffi_sift3B, :sift3B, [:string, :string, :uint, :double], :double

        # phonetic
        attach_function :ffi_soundex, :soundex, [:string], :pointer
        attach_function :ffi_refined_soundex, :refined_soundex, [:string], :pointer

        # helper to verify strings
        def is_string(object)
            unless object.kind_of? String
                raise TypeError,
                    "Wrong argument type #{object.class} (expected String)"
            end
        end
    end
end
{'fraction_non_alphanumeric': 0.09079365079365079, 'fraction_numerical': 0.017936507936507938, 'mean_word_length': 1.9735724398301084, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7114843', 'n_tokens_mistral': 2144, 'n_tokens_neox': 2071, 'n_words': 414}