<commit_before>require 'speeches'
require 'id'
require 'speech'

class HansardHeading
  def initialize
    @title = ""
    @subtitle = ""
  end
  
  def output(x, newtitle, newsubtitle, speech_id, url)
    # Only add headings if they have changed
    if newtitle != @title
      x.tag!("major-heading", newtitle, :id => speech_id, :url => url)
    end
    if newtitle != @title || newsubtitle != @subtitle
      x.tag!("minor-heading", newsubtitle, :id => speech_id, :url => url)
    end
    @title = newtitle
    @subtitle = newsubtitle
  end
end

class HansardParser
  
  def HansardParser.parse_date(date, xml_filename, people)
    conf = Configuration.new

    # Required to workaround long viewstates generated by .NET (whatever that means)
    # See http://code.whytheluckystiff.net/hpricot/ticket/13
    Hpricot.buffer_size = 262144

    agent = WWW::Mechanize.new
    agent.set_proxy(conf.proxy_host, conf.proxy_port)

    url = "http://parlinfoweb.aph.gov.au/piweb/browse.aspx?path=Chamber%20%3E%20House%20Hansard%20%3E%20#{date.year}%20%3E%20#{date.day}%20#{Date::MONTHNAMES[date.month]}%20#{date.year}"
    page = agent.get(url)

    parse_day_page(page, date, agent, people, xml_filename)
  end
  
  def HansardParser.parse_sub_day_speech_page(sub_page, x, heading, speech_id, people, date, time, url)
    newtitle = sub_page.search('div#contentstart div.hansardtitle').inner_html
    newsubtitle = sub_page.search('div#contentstart div.hansardsubtitle').inner_html

    heading.output(x, newtitle, newsubtitle, speech_id, url)

    speeches = Speeches.new

    # Untangle speeches from subspeeches
    speech_content = Hpricot::Elements.new
    content = sub_page.search('div#contentstart > div.speech0 > *')
    tag_classes = content.map{|e| e.attributes["class"]}
    subspeech0_index = tag_classes.index("subspeech0")
    paraitalic_index = tag_classes.index("paraitalic")

    if subspeech0_index.nil?
      subspeech_index = paraitalic_index
    elsif paraitalic_index.nil?
      subspeech_index = subspeech0_index
    else
      subspeech_index = min(subspeech0_index, paraitalic_index)
    end

    if subspeech_index
      speech_content = content[0..subspeech_index-1]
      subspeeches_content = content[subspeech_index..-1]
    else
      speech_content = content
    end
    # Extract speaker name from link
    speaker = extract_speaker_from_talkername_tag(speech_content, people, date)
    speeches.add_speech(speaker, time, url, speech_id, speech_content)

    if subspeeches_content
      process_subspeeches(subspeeches_content, people, date, speeches, time, url, speech_id, speaker)
    end
    speeches.write(x)   
  end
  
  def HansardParser.parse_sub_day_page(link_text, sub_page, x, heading, speech_id, people, date)
    # Only going to consider speeches for the time being
    if link_text =~ /Speech:/
      # Link text for speech has format:
      # HEADING > NAME > HOUR:MINS:SECS
      split = link_text.split('>').map{|a| a.strip}
      puts "Warning: Expected split to have length 3" unless split.size == 3
      time = split[2]
      # Extract permanent URL of this subpage. Also, quoting because there is a bug
      # in XML Builder that for some reason is not quoting attributes properly
      url = quote(sub_page.links.text("[Permalink]").uri.to_s)
      
      parse_sub_day_speech_page(sub_page, x, heading, speech_id, people, date, time, url)
    else
      puts "WARNING: Skipping: #{link_text}"
    end<commit_msg>Checking the kind of sub-page now<commit_after>require 'speeches'
require 'id'
require 'speech'

class HansardHeading
  def initialize
    @title = ""
    @subtitle = ""
  end
  
  def output(x, newtitle, newsubtitle, speech_id, url)
    # Only add headings if they have changed
    if newtitle != @title
      x.tag!("major-heading", newtitle, :id => speech_id, :url => url)
    end
    if newtitle != @title || newsubtitle != @subtitle
      x.tag!("minor-heading", newsubtitle, :id => speech_id, :url => url)
    end
    @title = newtitle
    @subtitle = newsubtitle
  end
end

class HansardParser
  
  def HansardParser.parse_date(date, xml_filename, people)
    conf = Configuration.new

    # Required to workaround long viewstates generated by .NET (whatever that means)
    # See http://code.whytheluckystiff.net/hpricot/ticket/13
    Hpricot.buffer_size = 262144

    agent = WWW::Mechanize.new
    agent.set_proxy(conf.proxy_host, conf.proxy_port)

    url = "http://parlinfoweb.aph.gov.au/piweb/browse.aspx?path=Chamber%20%3E%20House%20Hansard%20%3E%20#{date.year}%20%3E%20#{date.day}%20#{Date::MONTHNAMES[date.month]}%20#{date.year}"
    page = agent.get(url)

    parse_day_page(page, date, agent, people, xml_filename)
  end
  
  def HansardParser.parse_sub_day_speech_page(sub_page, x, heading, speech_id, people, date, time, url)
    newtitle = sub_page.search('div#contentstart div.hansardtitle').inner_html
    newsubtitle = sub_page.search('div#contentstart div.hansardsubtitle').inner_html

    heading.output(x, newtitle, newsubtitle, speech_id, url)

    speeches = Speeches.new

    # Untangle speeches from subspeeches
    speech_content = Hpricot::Elements.new
    content = sub_page.search('div#contentstart > div.speech0 > *')
    tag_classes = content.map{|e| e.attributes["class"]}
    subspeech0_index = tag_classes.index("subspeech0")
    paraitalic_index = tag_classes.index("paraitalic")

    if subspeech0_index.nil?
      subspeech_index = paraitalic_index
    elsif paraitalic_index.nil?
      subspeech_index = subspeech0_index
    else
      subspeech_index = min(subspeech0_index, paraitalic_index)
    end

    if subspeech_index
      speech_content = content[0..subspeech_index-1]
      subspeeches_content = content[subspeech_index..-1]
    else
      speech_content = content
    end
    # Extract speaker name from link
    speaker = extract_speaker_from_talkername_tag(speech_content, people, date)
    speeches.add_speech(speaker, time, url, speech_id, speech_content)

    if subspeeches_content
      process_subspeeches(subspeeches_content, people, date, speeches, time, url, speech_id, speaker)
    end
    speeches.write(x)   
  end
  
  def HansardParser.parse_sub_day_page(link_text, sub_page, x, heading, speech_id, people, date)
    # Only going to consider speeches for the time being
    if link_text =~ /^Speech:/
      # Link text for speech has format:
      # HEADING > NAME > HOUR:MINS:SECS
      split = link_text.split('>').map{|a| a.strip}
      puts "Warning: Expected split to have length 3" unless split.size == 3
      time = split[2]
      # Extract permanent URL of this subpage. Also, quoting because there is a bug
      # in XML Builder that for some reason is not quoting attributes properly
      url = quote(sub_page.links.text("[Permalink]").uri.to_s)
      
      parse_sub_day_speech_page(sub_page, x, heading, speech_id, people, date, time, url)
    elsif link_text == "Official Hansard" || link_text =~ /^Start of Business/ || link_text == "Adjournment"
      # Do nothing - skip this entirely
    elsif link_text =~ /^Procedural text:/ || link_text =~ /^QUESTIONS WITHOUT NOTICE:/ || link_text =~ /^QUESTIONS IN WRITING:/ ||
      link_text =~ /^Division:/
      puts "WARNING: Not yet supporting: #{link_text}"
    else
      puts "WARNING: Unsupported: #{link_text}"
    end
{'fraction_non_alphanumeric': 0.10616812227074236, 'fraction_numerical': 0.01118995633187773, 'mean_word_length': 3.695067264573991, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4293915', 'n_tokens_mistral': 2578, 'n_tokens_neox': 2421, 'n_words': 569}