<commit_before>require_relative './pattern_counter.rb'

class Hansard
  BY_LAW_REGEX = /\d+\/\d{4}/
  CAPITALIZED_PHRASE_REGEX = /([A-Z][\w-]*(?:\s+[A-Z][\w-]+)+)/
  WORD_REGEX = /[\p{Alpha}\-']+/

  def initialize(options)
    @hansard_data  = options[:json_hansard]
    @stop_words    = options[:stop_words] || []
  end

  def all_sections
    @hansard_data['hansard']
  end

  def date_of_meeting
    @hansard_data['meta']['date']
  end

  def sorted_word_occurrences
    PatternCounter.new(all_words, WORD_REGEX, @stop_words)
                  .unique_matches_sorted
  end

  def by_laws_counted
    PatternCounter.new(all_words, BY_LAW_REGEX)
                  .unique_matches_sorted
  end

  def capitalized_phrases_counted
    excluded_phrases = attendance_with_guests
    PatternCounter.new(all_words, CAPITALIZED_PHRASE_REGEX, excluded_phrases)
                  .unique_matches_sorted
                  .map do |phrase, count| # Re-Capitalize Phrases
                    [phrase.split.map(&:capitalize).join(' '), count]
                  end
  end

  def speakers_sorted_by_counted_words
    counted_words_by_speaker.sort_by do |_name, counted_words|
      counted_words
    end.reverse!
  end

  private

  def sections_of_type(section_type)
    @hansard_data['hansard'].select do |section|
      section['type'] == section_type
    end
  end

  def speaker_sections
    @speaker_sections ||= sections_of_type('speaker')
  end

  def attendance_with_guests
    speaker_sections.map do |section|
      section['name']<commit_msg>Update hansard.rb<commit_after>require_relative './pattern_counter.rb'

class Hansard
  BY_LAW_REGEX = /\d+\/\d{4}/
  CAPITALIZED_PHRASE_REGEX = /([A-Z][\w-]*(?:\s+[A-Z][\w-]+)+)/
  WORD_REGEX = /[\p{Alpha}\-']+/

  def initialize(options)
    @hansard_data  = options[:json_hansard]
    @stop_words    = options[:stop_words] || []
  end

  def all_sections
    @hansard_data['hansard']
  end
  
  def speaker_sections
    @speaker_sections ||= sections_of_type('speaker')
  end
  
  def motion_sections
    @motion_sections ||= sections_of_type('motion')
  end
  
  def vote_sections
    @vote_sections ||= sections_of_type('vote')
  end
  
  def topic_sections
    @topic_sections ||= sections_of_type('section')
  end

  def date_of_meeting
    @hansard_data['meta']['date']
  end

  def sorted_word_occurrences
    PatternCounter.new(all_words, WORD_REGEX, @stop_words)
                  .unique_matches_sorted
  end

  def by_laws_counted
    PatternCounter.new(all_words, BY_LAW_REGEX)
                  .unique_matches_sorted
  end

  def capitalized_phrases_counted
    excluded_phrases = attendance_with_guests
    PatternCounter.new(all_words, CAPITALIZED_PHRASE_REGEX, excluded_phrases)
                  .unique_matches_sorted
                  .map do |phrase, count| # Re-Capitalize Phrases
                    [phrase.split.map(&:capitalize).join(' '), count]
                  end
  end

  def speakers_sorted_by_counted_words
    counted_words_by_speaker.sort_by do |_name, counted_words|
      counted_words
    end.reverse!
  end

  private

  def sections_of_type(section_type)
    @hansard_data['hansard'].select do |section|
      section['type'] == section_type
    end
  end

  def attendance_with_guests
    speaker_sections.map do |section|
      section['name']
{'fraction_non_alphanumeric': 0.10606971153846154, 'fraction_numerical': 0.0006009615384615385, 'mean_word_length': 3.069682151589242, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2321349', 'n_tokens_mistral': 1211, 'n_tokens_neox': 1162, 'n_words': 187}