<commit_before>require './lib/commons/parser'

class CommonsDebatesParser < CommonsParser
  attr_reader :component_name, :component_prefix
  
  def initialize(date, component_name="Debates and Oral Answers")
    super(date)
    @component_name = component_name
    @component_prefix = "d"
    @department = ""
    @bill = {}
    @subsection_name = ""
    @section_stack = []
  end
  
  def get_component_index
    super(component_name)
  end
  
  
  private
  
  def parse_node(node)
    case node.name
    when "a"
      process_links_and_columns(node)
      determine_fragment_type(node)
    when "h2"
      process_top_level_heading(minify_whitespace(node.text), node.content)
    when "h3"
      process_heading(minify_whitespace(node.text))
    when "h4"
      process_subheading(sanitize_text(minify_whitespace(node.text)))
    when "h5"
      process_timestamp(minify_whitespace(node.text))
    when "p"
      process_para(node)
    when "center"
      if @section_stack.last and @section_stack.last.title =~ /deferred division(?:s?)/i and !(node.text =~ /^Division No/)
        start_new_section
        @section = create_new_division(node.text)
      else
        process_para(node)
      end
    end
  end
  
  def process_top_level_heading(text, title)
    case text
    when "House of Commons"
      setup_preamble(title)
    when "Oral Answers to Questions"
      create_oral_answers_subsection(text)
    end
  end
  
  def process_heading(text)
    if (@page_fragment_type == "department heading" and @subsection_name == "Oral Answer")
      @department = text
      if text.downcase != "prayers" and @section
        start_new_section
        @section = create_new_container(text, @section_stack.last)
      else
        if @section.type == "Preamble"
          #wait, is that even possible?
          create_new_noncontribution_para(text)
        else
          start_new_section
          setup_preamble(text)
        end
      end
    elsif @page_fragment_type == "subject heading" and @subsection_name == "Oral Answer"
      if text == "Topical Questions"
        start_subsection
        @subsection_name = "Topical Questions"
        @section = create_new_container(sanitize_text(text), @section_stack.last)
      else
        #a subject heading, inside Oral Answers
        start_new_section
        @section = create_new_question(sanitize_text(text))
      end
      @section.sequence = @section_seq
      @section.columns = [@column]
      @section.department if @department
    else
      if @subsection_name == "Oral Answer" or @subsection_name == "Topical Questions"
        save_section
      end
      if text =~ /.? Bill(?: |$)/
        @bill[:title] = text.gsub(" [Lords]", "")
      end
      
      @subsection_name = ""
      if text.downcase == "prayers"
        start_new_section
        setup_preamble(text)
      else
        start_new_section
        if text =~ /deferred division(?:s?)/i
          @section = create_new_container(text, @section_stack.last)
        else
          @section = create_new_debate(sanitize_text(text))
        end
      end
    end
  end
  
  def process_subheading(text)
    if @section and @section.type == "Preamble"
      create_new_noncontribution_para(text)
    else
      if text.downcase =~ /^back\s?bench business$/ \
          or text.downcase =~ /^business without debate$/i
        #treat as honorary h3 / main heading
        @subsection_name = "Backbench Business"
        start_subsection
        @section = create_new_container(text)
      else
        para = create_new_noncontribution_para(sanitize_text(text))
        @subject = sanitize_text(text)
      end
    end
  end
  
  def create_new_debate(text)
    @section_seq += 1
    section_ident = "#{@hansard_component.ident}_#{@section_seq.to_s.rjust(6, "0")}"
    section = Debate.find_or_create_by(ident: section_ident)
    section.title = text
    section.url = "#{@page.url}\##{@last_link}"
    section.sequence = @section_seq
    section.columns = [@column]
    section.component = @hansard_component
    @para_seq = 0
    section
  end
  
  def create_new_division(title=nil)
    @tellers = false
    @current_list = ""
    
    @section_seq += 1
    section_ident = "#{@hansard_component.ident}_#{@section_seq.to_s.rjust(6, "0")}"
    section = Division.find_or_create_by(ident: section_ident)
    if title
      section.title = "Division - #{title}"
    else
      section.title = "Division - #{@section.title}" #borrow the previous section's title, certain assumption there
    end
    section.url = "#{@page.url}\##{@last_link}"
    section.ayes = []
    section.noes = []
    section.tellers_ayes = []
    section.tellers_noes = []
    section.component = @hansard_component
    section.columns = [@column]
    section.sequence = @section_seq
    section.members = []
    @para_seq = 0
    section
  end
  
  def create_new_container(title, parent=nil)
    @section_seq += 1
    section_ident = "#{@hansard_component.ident}_#{@section_seq.to_s.rjust(6, "0")}"
    section = Container.find_or_create_by(ident: section_ident)
    section.url = "#{@page.url}\##{@last_link}"
    section.component = @hansard_component
    section.title = title
    section.sequence = @section_seq
    section.columns = [@column]
    @para_seq = 0
    if parent
      section.parent_section = parent
      parent.sections << section unless parent.sections.include?(section)
      @section_stack << section
    else
      @section_stack = [section]
    end
    section
  end
  
  def create_new_question(text, member=false)
    @section_seq += 1
    section_ident = "#{@hansard_component.ident}_#{@section_seq.to_s.rjust(6, "0")}"
    section = Question.find_or_create_by(ident: section_ident)
    section.question_type = "for oral answer"
    if section.number.blank?
      number = get_question_number(text)
      unless number.blank?
        section.number = number
      end
    end
    section.asked_by = member.index_name if member
    if text =~ /^\s?(T\d+).(?: |\[)/
      section.title = "Topical Questions - #{$1}"
    elsif text =~ /^\s?(\d+).(?: |\[)/ #rare, weird format
      section.title = "Question - #{$1}"
    else
      section.title = text
    end
    section.sequence = @section_seq
    section.columns = [@column]
    section.component = @hansard_component
    section.department = @department if @department
    section.url = "#{@page.url}\##{@last_link}"
    @para_seq = 0
    section
  end
  
  def process_division(text)
    case text.strip
    when /^The House (having )?divided/, /^Motion/, /^Question/, /^Bill accordingly/
      @current_list = ""
      @tellers = false
      para = create_new_noncontribution_para(sanitize_text(text))
    when /^Ayes\s\d+,\s?Noes\s\d+/
      para = create_new_noncontribution_para(sanitize_text(text))
    when /^Division No\. ([^\]]*)\]/
      @section.number = $1
    when /\[(\d+\.\d+ (a|p)m)/
      process_timestamp($1)
    when "AYES"
      @current_list = "ayes"<commit_msg>mopping up a few more oddities within Debates<commit_after>require './lib/commons/parser'

class CommonsDebatesParser < CommonsParser
  attr_reader :component_name, :component_prefix
  
  def initialize(date, component_name="Debates and Oral Answers")
    super(date)
    @component_name = component_name
    @component_prefix = "d"
    @department = ""
    @bill = {}
    @subsection_name = ""
    @section_stack = []
  end
  
  def get_component_index
    super(component_name)
  end
  
  
  private
  
  def parse_node(node)
    case node.name
    when "a"
      process_links_and_columns(node)
      determine_fragment_type(node)
    when "h2"
      process_top_level_heading(minify_whitespace(node.text), node.content)
    when "h3"
      if node.text =~ /^That this House takes note/
        process_para(node)
      else
        process_heading(minify_whitespace(node.text))
      end
    when "h4"
      process_subheading(sanitize_text(minify_whitespace(node.text)))
    when "h5"
      process_timestamp(minify_whitespace(node.text))
    when "p"
      process_para(node)
    when "center"
      if @section_stack.last and @section_stack.last.title =~ /deferred division(?:s?)/i and !(node.text =~ /^Division No/)
        start_new_section
        @section = create_new_division(node.text)
      else
        process_para(node)
      end
    end
  end
  
  def process_top_level_heading(text, title)
    case text
    when "House of Commons"
      setup_preamble(title)
    when "Oral Answers to Questions"
      create_oral_answers_subsection(text)
    end
  end
  
  def process_heading(text)
    if (@page_fragment_type == "department heading" and @subsection_name == "Oral Answer")
      @department = text
      if text.downcase != "prayers" and @section
        start_new_section
        @section = create_new_container(text, @section_stack.last)
      else
        if @section.type == "Preamble"
          #wait, is that even possible?
          create_new_noncontribution_para(text)
        else
          start_new_section
          setup_preamble(text)
        end
      end
    elsif @page_fragment_type == "subject heading" and @subsection_name == "Oral Answer"
      if text == "Topical Questions"
        start_subsection
        @subsection_name = "Topical Questions"
        @section = create_new_container(sanitize_text(text), @section_stack.last)
      else
        #a subject heading, inside Oral Answers
        start_new_section
        @section = create_new_question(sanitize_text(text))
      end
      @section.sequence = @section_seq
      @section.columns = [@column]
      @section.department if @department
    else
      if @subsection_name == "Oral Answer" or @subsection_name == "Topical Questions"
        save_section
      end
      if text =~ /.? Bill(?: |$)/
        @bill[:title] = text.gsub(" [Lords]", "")
      end
      
      @subsection_name = ""
      if text.downcase == "prayers"
        start_new_section
        setup_preamble(text)
      else
        start_new_section
        if text =~ /deferred division(?:s?)/i
          @section = create_new_container(text, @section_stack.last)
        else
          @section = create_new_debate(sanitize_text(text))
        end
      end
    end
  end
  
  def process_subheading(text)
    if @section and @section.type == "Preamble"
      create_new_noncontribution_para(text)
    else
      if text.downcase =~ /^back\s?bench business$/ \
          or text.downcase =~ /^business without debate$/i
        #treat as honorary h3 / main heading
        @subsection_name = "Backbench Business"
        start_subsection
        @section = create_new_container(text)
      else
        para = create_new_noncontribution_para(sanitize_text(text))
        @subject = sanitize_text(text)
      end
    end
  end
  
  def create_new_debate(text)
    @section_seq += 1
    section_ident = "#{@hansard_component.ident}_#{@section_seq.to_s.rjust(6, "0")}"
    section = Debate.find_or_create_by(ident: section_ident)
    section.title = text
    section.url = "#{@page.url}\##{@last_link}"
    section.sequence = @section_seq
    section.columns = [@column]
    section.component = @hansard_component
    @para_seq = 0
    section
  end
  
  def create_new_division(title=nil)
    @tellers = false
    @current_list = ""
    
    @section_seq += 1
    section_ident = "#{@hansard_component.ident}_#{@section_seq.to_s.rjust(6, "0")}"
    section = Division.find_or_create_by(ident: section_ident)
    if title
      section.title = "Division - #{title}"
    else
      section.title = "Division - #{@section.title}" #borrow the previous section's title, certain assumption there
    end
    section.url = "#{@page.url}\##{@last_link}"
    section.ayes = []
    section.noes = []
    section.tellers_ayes = []
    section.tellers_noes = []
    section.component = @hansard_component
    section.columns = [@column]
    section.sequence = @section_seq
    section.members = []
    @para_seq = 0
    section
  end
  
  def create_new_container(title, parent=nil)
    @section_seq += 1
    section_ident = "#{@hansard_component.ident}_#{@section_seq.to_s.rjust(6, "0")}"
    section = Container.find_or_create_by(ident: section_ident)
    section.url = "#{@page.url}\##{@last_link}"
    section.component = @hansard_component
    section.title = title
    section.sequence = @section_seq
    section.columns = [@column]
    @para_seq = 0
    if parent
      section.parent_section = parent
      parent.sections << section unless parent.sections.include?(section)
      @section_stack << section
    else
      @section_stack = [section]
    end
    section
  end
  
  def create_new_question(text, member=false)
    @section_seq += 1
    section_ident = "#{@hansard_component.ident}_#{@section_seq.to_s.rjust(6, "0")}"
    section = Question.find_or_create_by(ident: section_ident)
    section.question_type = "for oral answer"
    if section.number.blank?
      number = get_question_number(text)
      unless number.blank?
        section.number = number
      end
    end
    section.asked_by = member.index_name if member
    if text =~ /^\s?(T\d+)\s?.(?: |\[)/
      section.title = "Topical Questions - #{$1}"
    elsif text =~ /^\s?(\d+).(?: |\[)/ #rare, weird format
      section.title = "Question - #{$1}"
    else
      section.title = text
    end
    section.sequence = @section_seq
    section.columns = [@column]
    section.component = @hansard_component
    section.department = @department if @department
    section.url = "#{@page.url}\##{@last_link}"
    @para_seq = 0
    section
  end
  
  def process_division(text)
    case text.strip
    when /^The House (having )?divided/, /^Motion/, /^Question/, /^Bill accordingly/
      @current_list = ""
      @tellers = false
      para = create_new_noncontribution_para(sanitize_text(text))
    when /^Ayes\s\d+,\s?Noes\s\d+/
      para = create_new_noncontribution_para(sanitize_text(text))
    when /^Division No\. ([^\]]*)\]/
      @section.number = $1
    when /\[(\d+\.\d+ (a|p)m)/
      process_timestamp($1)
    when "AYES"
      @current_list = "ayes"
{'fraction_non_alphanumeric': 0.11627741889685525, 'fraction_numerical': 0.003549371761198268, 'mean_word_length': 2.8003776638791478, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6175000', 'n_tokens_mistral': 4656, 'n_tokens_neox': 4601, 'n_words': 922}