<commit_before># encoding: UTF-8
require "gsub_filter/version"

class GsubFilter
  attr_reader :stocks, :filters
  def initialize(input=nil)
    @input = input
    @filters = []
    @stocks = Hash.new{ |h,k| h[k] = [] }
  end

  # default opt: {global: true, at: -1, replace: false}
  def filter(pattern, opt={}, &replace)
    opt = opt_parse_for_filter(opt)
    @filters[opt[:at], opt[:replace]] = [[pattern, replace, opt[:global]]]
    nil
  end

  def replace(at, pattern, opt={}, &replace)
    opt = {at: at, replace: true}.update(opt)
    filter(pattern, opt, &replace)
  end

  def run(str=nil)
    str ||= @input
    @filters.compact.each do |pattern, replace, meth|
      str = str.send(meth, pattern) { replace[$~, @stocks] }
    end
    str
  end

  def opt_clear
    @stocks.clear
  end

  private
  def opt_parse_for_filter(opt)
    opt = {global: true, at: -1, replace: false}.update(opt)
    opt[:global] = opt[:global] ? :gsub : :sub
    opt[:replace] = opt[:replace] ? 1 : 0
    if opt[:replace].zero? && opt[:at] < 0
      opt[:at] = @filters.size + opt[:at] + 1
    end
    opt
  end<commit_msg>fix method name for clear stocks hash<commit_after># encoding: UTF-8
require "gsub_filter/version"

class GsubFilter
  attr_reader :stocks, :filters
  def initialize(input=nil)
    @input = input
    @filters = []
    @stocks = Hash.new{ |h,k| h[k] = [] }
  end

  # default opt: {global: true, at: -1, replace: false}
  def filter(pattern, opt={}, &replace)
    opt = opt_parse_for_filter(opt)
    @filters[opt[:at], opt[:replace]] = [[pattern, replace, opt[:global]]]
    nil
  end

  def replace(at, pattern, opt={}, &replace)
    opt = {at: at, replace: true}.update(opt)
    filter(pattern, opt, &replace)
  end

  def run(str=nil)
    str ||= @input
    @filters.compact.each do |pattern, replace, meth|
      str = str.send(meth, pattern) { replace[$~, @stocks] }
    end
    str
  end

  def clear_stocks
    @stocks.clear
  end

  private
  def opt_parse_for_filter(opt)
    opt = {global: true, at: -1, replace: false}.update(opt)
    opt[:global] = opt[:global] ? :gsub : :sub
    opt[:replace] = opt[:replace] ? 1 : 0
    if opt[:replace].zero? && opt[:at] < 0
      opt[:at] = @filters.size + opt[:at] + 1
    end
    opt
  end
{'fraction_non_alphanumeric': 0.16245006657789615, 'fraction_numerical': 0.006213936972924989, 'mean_word_length': 3.151012891344383, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2439771', 'n_tokens_mistral': 838, 'n_tokens_neox': 811, 'n_words': 215}