<commit_before>module OverSIP

  module Utils

    # It ensures that two identical byte secuences are matched regardless
    # they have different encoding.
    # For example in Ruby the following returns false:
    #   "iñaki".force_encoding(::Encoding::BINARY) == "iñaki"
    def self.string_compare string1, string2
      string1.force_encoding(::Encoding::BINARY) == string2.force_encoding(::Encoding::BINARY)
    end

    # This avoid "invalid byte sequence in UTF-8" when the directly doing:
    #   string =~ /EXPRESSION/
    # and string has invalid UTF-8 bytes secuence.
    # NOTE: expression argument must be a Regexp expression (with / symbols at the
    # begining and at the end).
    def self.regexp_compare string, expression
      return false  unless string && string.valid_encoding?
      string.force_encoding(::Encoding::BINARY) =~ expression
    end

  end

end<commit_msg>Improvements in OverSIP::Utils module methods.<commit_after>module OverSIP

  module Utils

    # It ensures that two identical byte secuences are matched regardless
    # they have different encoding.
    # For example in Ruby the following returns false:
    #   "iñaki".force_encoding(::Encoding::BINARY) == "iñaki"
    def self.string_compare string1, string2
      string1.to_s.force_encoding(::Encoding::BINARY) == string2.to_s.force_encoding(::Encoding::BINARY)
    end

    # This avoid "invalid byte sequence in UTF-8" when the directly doing:
    #   string =~ /EXPRESSION/
    # and string has invalid UTF-8 bytes secuence.
    # NOTE: expression argument must be a Regexp expression (with / symbols at the
    # begining and at the end).
    def self.regexp_compare string, expression
      string = string.to_s
      return false  unless string.valid_encoding?
      string.force_encoding(::Encoding::BINARY) =~ expression
    end

  end

end
{'fraction_non_alphanumeric': 0.08378378378378379, 'fraction_numerical': 0.006486486486486486, 'mean_word_length': 3.471014492753623, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6470596', 'n_tokens_mistral': 539, 'n_tokens_neox': 525, 'n_words': 176}