<commit_before>class Hansard
  BY_LAW_REGEX = /\d+\/\d{4}/
  CAPITALIZED_PHRASE_REGEX = /([A-Z][\w-]*(?:\s+[A-Z][\w-]+)+)/
  WORD_REGEX = /[\p{Alpha}\-']+/

  def initialize(options)
    @hansard_data = options[:json_hansard]
    @stop_words =  options[:stop_words] || []
  end

  def all_sections
    @hansard_data['hansard']
  end

  def date_of_meeting
    @hansard_data['meta']['date']
  end

  def sorted_word_occurrences
    Mentions.count(all_words, WORD_REGEX, @stop_words)
  end

  def by_laws_counted
    Mentions.count(all_words, BY_LAW_REGEX)
  end

  def capitalized_phrases_counted
    Mentions.count(all_words, CAPITALIZED_PHRASE_REGEX, attendance_with_guests)
  end

  def speakers_sorted_by_counted_words
    counted_words_by_speaker.map do |name, counted_words|
      { 'name' => name, 'word_count' => counted_words.word_count}
    end.sort { |a, b| b['word_count'] <=> a['word_count'] }
  end

  private
  def sections_of_type(section_type)
    @hansard_data['hansard'].select { |section| section['type'] == section_type }
  end

  def speaker_sections
    @speaker_sections ||= sections_of_type('speaker')
  end

  def attendance_with_guests
    @attendance_with_guests ||= speaker_sections.map do |section|
      section['name']
    end.uniq
  end

  def all_words
    @all_words ||= speaker_sections.reduce('') do |all_words, speaker_section|
      all_words + speaker_section['spoken']
    end
  end

  def all_words_by_speaker
    @all_words_by_seaker ||= speaker_sections.reduce(Hash.new('')) do |h, section|
      h[section['name']] += section['spoken']
      h
    end
  end

  def counted_words_by_speaker
    @counted_words_by_speaker ||= all_words_by_speaker.each_with_object({}) do |(k, v), h|
      h[k] = WordsCounted.count(v, exclude: @stop_words)
    end
  end
end

class Mentions
  def initialize(all_words, regex, excludes = [])
    @all_words = all_words
    @regex = regex
    @excludes = excludes.map(&:downcase)
  end

  def self.count(all_words, regex, excludes = [])
    new(all_words, regex, excludes).mentions_counted_and_sorted
  end

  def mentions_counted_and_sorted
    unique_mentions.map do |mention|
      { 'mention' => mention, 'count' => mentions.count { |m| m == mention } }
    end.sort{ |a, b| b['count'] <=> a['count']}
  end

  private
  def unique_mentions
    @unique_mentions ||= mentions.uniq
  end

  def mentions
    return @mentions  if @mentions
    @mentions = @all_words.scan(@regex)
    @mentions = remove_one_level_of_array_nesting(@mentions)
    @mentions = @mentions.map(&:downcase).reject { |mention| @excludes.include?(mention.downcase) }
  end

  def remove_one_level_of_array_nesting(object)
    object.first.is_a?(Array) ? object.map(&:first) : object
  end
end
<commit_msg>Each with Object is cleaner than reduce for Hash creation.<commit_after>class Hansard
  BY_LAW_REGEX = /\d+\/\d{4}/
  CAPITALIZED_PHRASE_REGEX = /([A-Z][\w-]*(?:\s+[A-Z][\w-]+)+)/
  WORD_REGEX = /[\p{Alpha}\-']+/

  def initialize(options)
    @hansard_data = options[:json_hansard]
    @stop_words =  options[:stop_words] || []
  end

  def all_sections
    @hansard_data['hansard']
  end

  def date_of_meeting
    @hansard_data['meta']['date']
  end

  def sorted_word_occurrences
    Mentions.count(all_words, WORD_REGEX, @stop_words)
  end

  def by_laws_counted
    Mentions.count(all_words, BY_LAW_REGEX)
  end

  def capitalized_phrases_counted
    Mentions.count(all_words, CAPITALIZED_PHRASE_REGEX, attendance_with_guests)
  end

  def speakers_sorted_by_counted_words
    counted_words_by_speaker.map do |name, counted_words|
      { 'name' => name, 'word_count' => counted_words.word_count}
    end.sort { |a, b| b['word_count'] <=> a['word_count'] }
  end

  private
  def sections_of_type(section_type)
    @hansard_data['hansard'].select { |section| section['type'] == section_type }
  end

  def speaker_sections
    @speaker_sections ||= sections_of_type('speaker')
  end

  def attendance_with_guests
    @attendance_with_guests ||= speaker_sections.map do |section|
      section['name']
    end.uniq
  end

  def all_words
    @all_words ||= speaker_sections.reduce('') do |all_words, speaker_section|
      all_words + speaker_section['spoken']
    end
  end

  def all_words_by_speaker
    @all_words_by_seaker ||= speaker_sections.each_with_object(Hash.new('')) do |section, h|
      h[section['name']] += section['spoken']
    end
  end

  def counted_words_by_speaker
    @counted_words_by_speaker ||= all_words_by_speaker.each_with_object({}) do |(k, v), h|
      h[k] = WordsCounted.count(v, exclude: @stop_words)
    end
  end
end

class Mentions
  def initialize(all_words, regex, excludes = [])
    @all_words = all_words
    @regex = regex
    @excludes = excludes.map(&:downcase)
  end

  def self.count(all_words, regex, excludes = [])
    new(all_words, regex, excludes).mentions_counted_and_sorted
  end

  def mentions_counted_and_sorted
    unique_mentions.map do |mention|
      { 'mention' => mention, 'count' => mentions.count { |m| m == mention } }
    end.sort{ |a, b| b['count'] <=> a['count']}
  end

  private
  def unique_mentions
    @unique_mentions ||= mentions.uniq
  end

  def mentions
    return @mentions  if @mentions
    @mentions = @all_words.scan(@regex)
    @mentions = remove_one_level_of_array_nesting(@mentions)
    @mentions = @mentions.map(&:downcase).reject { |mention| @excludes.include?(mention.downcase) }
  end

  def remove_one_level_of_array_nesting(object)
    object.first.is_a?(Array) ? object.map(&:first) : object
  end
end

{'fraction_non_alphanumeric': 0.12911939492166397, 'fraction_numerical': 0.0003601656762110571, 'mean_word_length': 4.012635379061372, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3888232', 'n_tokens_mistral': 2136, 'n_tokens_neox': 2054, 'n_words': 374}