<commit_before>require 'set'

Normalizr.configure do
  default :strip, :blank

  add :blank do |value|
    value unless /\A[[:space:]]*\z/ === value
  end

  TRUE_VALUES = [true, 1, '1', 't', 'T', 'true', 'TRUE', 'on', 'ON'].to_set

  add :boolean do |value|
    unless String === value && value.blank?
      TRUE_VALUES.include?(value)
    end
  end

  add :control_chars do |value|
    value.gsub!(/[[:cntrl:]&&[^[:space:]]]/, '') if String === value
    value
  end

  add :phone do |value|
    if String === value
      value.gsub!(/[^0-9]+/, '')
      value unless value.empty?
    else
      value
    end
  end

  add :squish do |value|
    if String === value
      value.strip!
      value.gsub!(/\s+/, ' ')
    end
    value
  end

  add :whitespace do |value|
    if String === value
      value.gsub!(/[^\S\n]+/, ' ')
      value.gsub!(/\s?\n\s?/, "\n")
      value.strip!
    end
    value
  end

  [:capitalize, :downcase, :strip, :upcase].each do |name|
    add name do |value|
      value.send(:"#{name}!") if String === value
      value
    end
  end
end
<commit_msg>normalizer should not modify passed-in parameter<commit_after>require 'set'

Normalizr.configure do
  default :strip, :blank

  add :blank do |value|
    value unless /\A[[:space:]]*\z/ === value
  end

  TRUE_VALUES = [true, 1, '1', 't', 'T', 'true', 'TRUE', 'on', 'ON'].to_set

  add :boolean do |value|
    unless String === value && value.blank?
      TRUE_VALUES.include?(value)
    end
  end

  add :control_chars do |value|
    if String === value
      value.gsub(/[[:cntrl:]&&[^[:space:]]]/, '')
    else
      value
    end
  end

  add :phone do |value|
    if String === value
      value = value.gsub(/[^0-9]+/, '')
      value unless value.empty?
    else
      value
    end
  end

  add :squish do |value|
    if String === value
      value.strip.gsub(/\s+/, ' ')
    else
      value
    end
  end

  add :whitespace do |value|
    if String === value
      value.gsub(/[^\S\n]+/, ' ').gsub(/\s?\n\s?/, "\n").strip
    else
      value
    end
  end

  [:capitalize, :downcase, :strip, :upcase].each do |name|
    add name do |value|
      if String === value
        value.send(:"#{name}")
      else
        value
      end
    end
  end
end

{'fraction_non_alphanumeric': 0.17712505562972852, 'fraction_numerical': 0.0035603026257231864, 'mean_word_length': 2.2114285714285713, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7363552', 'n_tokens_mistral': 832, 'n_tokens_neox': 804, 'n_words': 222}