<commit_before># -*- coding: utf-8 -*-
module Agate
  class Parser
    # Default options
    DEFAULTS = {
      :delimiters => "【】",
      :formatter  => :html
    }

    # Regexp reserved characters to escape when matching
    RESERVED = ["(", ")", "[", "]", "{", "}", ".", ",", "+", "*"]

    def initialize(options = {})
      @options = DEFAULTS.merge(options)

      @formatter = case @options[:formatter]
        when :html
          Agate::Formatter::HTML.new
        else
          Agate::Formatter::HTML.new
        end
    end

    # Parse `text` and return it with ruby character markup
    def parse(text)
      first = @options[:delimiters].chars.first
      last  = @options[:delimiters].chars.last

      first = /#{'\\' + first}/ if RESERVED.include? first
      last  = /#{'\\' + last}/  if RESERVED.include? last

      text.gsub(/(\p{Han}+)(#{first})([\p{Hiragana}\p{Katakana}]+)(#{last})/) { |match| @formatter.format($~) }
    end
  end
end<commit_msg>String#chars behavior differs between 1.9, 2.0 (fixes #9)<commit_after># -*- coding: utf-8 -*-
module Agate
  class Parser
    # Default options
    DEFAULTS = {
      :delimiters => "【】",
      :formatter  => :html
    }

    # Regexp reserved characters to escape when matching
    RESERVED = ["(", ")", "[", "]", "{", "}", ".", ",", "+", "*"]

    def initialize(options = {})
      @options = DEFAULTS.merge(options)

      @formatter = case @options[:formatter]
        when :html
          Agate::Formatter::HTML.new
        else
          Agate::Formatter::HTML.new
        end
    end

    # Parse `text` and return it with ruby character markup
    def parse(text)
      first = @options[:delimiters][0]
      last  = @options[:delimiters][-1]

      first = /#{'\\' + first}/ if RESERVED.include? first
      last  = /#{'\\' + last}/  if RESERVED.include? last

      text.gsub(/(\p{Han}+)(#{first})([\p{Hiragana}\p{Katakana}]+)(#{last})/) { |match| @formatter.format($~) }
    end
  end
end
{'fraction_non_alphanumeric': 0.19148936170212766, 'fraction_numerical': 0.004559270516717325, 'mean_word_length': 2.4467713787085517, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4359149', 'n_tokens_mistral': 669, 'n_tokens_neox': 650, 'n_words': 140}