<commit_before>class Frequencies
  def initialize(raw)
    @table = {}
    
    raw.each_line do |line|
      frequency = line.split(' ')[0].to_i
      pattern   = line.split(' ')[1..-1].join(' ')
      
      @table[pattern] = frequency
    end
  end
  
  def each &iterator
    @table.each &iterator
  end

  def probability_of(pattern)
    frequency_of(pattern) / total
  end
  
private
  def total
    @table.values.inject(0) {|sum, element| sum += element }
  end
  
  def frequency_of(pattern)
    @table[pattern].to_f
  end
end<commit_msg>Cache total to improve performance.<commit_after>class Frequencies
  def initialize(raw)
    @table = {}
    
    raw.each_line do |line|
      frequency = line.split(' ')[0].to_i
      pattern   = line.split(' ')[1..-1].join(' ')
      
      @table[pattern] = frequency
    end
    
    @total = @table.values.inject(0) {|sum, element| sum += element }
  end
  
  def each &iterator
    @table.each &iterator
  end

  def probability_of(pattern)
    frequency_of(pattern) / @total
  end
  
private  
  def frequency_of(pattern)
    @table[pattern].to_f
  end
end
{'fraction_non_alphanumeric': 0.12612612612612611, 'fraction_numerical': 0.007207207207207207, 'mean_word_length': 2.549520766773163, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4509128', 'n_tokens_mistral': 381, 'n_tokens_neox': 378, 'n_words': 87}