<commit_before>require 'cloze/japanese_string_helpers'

require 'active_support/core_ext/array/extract_options'

module Cloze::Deletion
  def cloze_delete(*args)
    opts = args.select {|arg| arg.is_a? Symbol}
    char = args.extract_options![:with] || Cloze::Deletion.default_deletion_character
    # TODO ^ better var name

    strs = []

    if opts.include? :each
      # TODO extract to method
      self.split('').each_with_index do |_, i|
        str    = self.clone
        str[i] = char
        strs  << str
      end
    end

    if opts.include? :all
      strs << Array.new(self.size, char).join
    end

    if opts.include? :kanji
      self.split('').each_with_index do |ch, i|
        next unless ch.kanji?
        str    = self.clone
        str[i] = char
        strs  << str
      end
    end

    strs
  end

private
  def self.default_deletion_character
    '#'
  end

  def self.included(base)
    raise TypeError, "#{self} must be included in String or one of its subclasses" unless base <= String
    base.send :include, Cloze::JapaneseStringHelpers
  end
end
<commit_msg>set default deletion character, all specs pass<commit_after>require 'cloze/japanese_string_helpers'

require 'active_support/core_ext/array/extract_options'
require 'active_support/core_ext/module/attribute_accessors'

module Cloze::Deletion
  mattr_accessor :default_deletion_character
  self.default_deletion_character = '#'

  def cloze_delete(*args)
    opts = args.select {|arg| arg.is_a? Symbol}
    char = args.extract_options![:with] || Cloze::Deletion.default_deletion_character
    # TODO ^ better var name

    strs = []

    if opts.include? :each
      # TODO extract to method
      self.split('').each_with_index do |_, i|
        str    = self.clone
        str[i] = char
        strs  << str
      end
    end

    if opts.include? :all
      strs << Array.new(self.size, char).join
    end

    if opts.include? :kanji
      self.split('').each_with_index do |ch, i|
        next unless ch.kanji?
        str    = self.clone
        str[i] = char
        strs  << str
      end
    end

    strs
  end

private
  def self.included(base)
    raise TypeError, "#{self} must be included in String or one of its subclasses" unless base <= String
    base.send :include, Cloze::JapaneseStringHelpers
  end
end

{'fraction_non_alphanumeric': 0.10004312203536007, 'fraction_numerical': 0.0, 'mean_word_length': 2.665086887835703, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5485226', 'n_tokens_mistral': 782, 'n_tokens_neox': 754, 'n_words': 194}