<commit_before>class ReferentValue < ActiveRecord::Base
  belongs_to :referent, :include => :referent_values

  # Class method to normalize a string for normalized_value attribute. 
  # Right now normalization is just downcasing. Only
  # metadata values should be normalized (ie, not 'identifier' or 'format').
  # identifier and format shoudl be stored in normalized_value unchanged.
  def self.normalize(input)
      # 'mb_chars' is neccesary for unicode.
      # normalized_value column only holds 254 bytes.. 
      return input.mb_chars.downcase.to_s[0..254]
  end
  
end
<commit_msg>Update referent values for mass assignment issue/requirement in recent rails 3 versions.<commit_after>class ReferentValue < ActiveRecord::Base
  attr_accessible :key_name, :value, :normalized_value, :metadata, :private_data
  belongs_to :referent, :include => :referent_values

  # Class method to normalize a string for normalized_value attribute. 
  # Right now normalization is just downcasing. Only
  # metadata values should be normalized (ie, not 'identifier' or 'format').
  # identifier and format shoudl be stored in normalized_value unchanged.
  def self.normalize(input)
      # 'mb_chars' is neccesary for unicode.
      # normalized_value column only holds 254 bytes.. 
      return input.mb_chars.downcase.to_s[0..254]
  end
  
end

{'fraction_non_alphanumeric': 0.07410179640718563, 'fraction_numerical': 0.01122754491017964, 'mean_word_length': 4.182170542635659, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1655741', 'n_tokens_mistral': 397, 'n_tokens_neox': 371, 'n_words': 135}