<commit_before>require 'speech'
require 'mechanize_proxy'
require 'configuration'
require 'debates'
require 'builder_alpha_attributes'
require 'house'
require 'people_image_downloader'

class UnknownSpeaker
  def initialize(name)
    @name = name
  end
  
  def id
    "unknown"
  end
  
  def name
    Name.title_first_last(@name)
  end
end

require 'rubygems'
require 'log4r'

class HansardParser
  attr_reader :logger
  
  def initialize(people)
    @people = people
    conf = Configuration.new
    
    # Set up logging
    @logger = Log4r::Logger.new 'HansardParser'
    # Log to both standard out and the file set in configuration.yml
    @logger.add(Log4r::Outputter.stdout)
    @logger.add(Log4r::FileOutputter.new('foo', :filename => conf.log_path, :trunc => false,
      :formatter => Log4r::PatternFormatter.new(:pattern => "[%l] %d :: %M")))

    bio = PeopleImageDownloader.new
    bio.attach_aph_person_ids(people)
  end
  
  def parse_date_house(date, xml_filename, house)
    @logger.info "Parsing #{house} speeches for #{date.strftime('%a %d %b %Y')}..."
    url = "http://parlinfoweb.aph.gov.au/piweb/browse.aspx?path=Chamber%20%3E%20#{house.representatives? ? "House" : "Senate"}%20Hansard%20%3E%20#{date.year}%20%3E%20#{date.day}%20#{Date::MONTHNAMES[date.month]}%20#{date.year}"

    debates = Debates.new(date, house)
    
    # Required to workaround long viewstates generated by .NET (whatever that means)
    # See http://code.whytheluckystiff.net/hpricot/ticket/13
    Hpricot.buffer_size = 1600000

    agent = MechanizeProxy.new
    agent.cache_subdirectory = date.to_s

    begin
      page = agent.get(url)
      # HACK: Don't know why if the page isn't found a return code isn't returned. So, hacking around this.
      if page.title == "ParlInfo Web - Error"
        throw "ParlInfo Web - Error"
      end
    rescue
      logger.warn "Could not retrieve overview page for date #{date}"
      return
    end
    # Structure of the page is such that we are only interested in some of the links
    page.links[30..-4].each do |link|
      begin
        parse_sub_day_page(link.to_s, agent.click(link), debates, date, house)
      rescue
        logger.error "Exception thrown during processing of sub page: #{link}"
        raise $!
      end
      # This ensures that every sub day page has a different major count which limits the impact
      # of when we start supporting things like written questions, procedurial text, etc..
      debates.increment_major_count
    end
    
    debates.output(xml_filename)
  end
  
  def parse_sub_day_page(link_text, sub_page, debates, date, house)
    # Only going to consider speeches for the time being
    if link_text =~ /^Speech:/ || link_text =~ /^QUESTIONS? WITHOUT NOTICE/i || link_text =~ /^QUESTIONS TO THE SPEAKER:/
      # Link text for speech has format:
      # HEADING > NAME > HOUR:MINS:SECS
      split = link_text.split('>').map{|a| a.strip}
      logger.error "Expected split to have length 3 in link text: #{link_text}" unless split.size == 3
      time = split[2]
      # Check that time is something valid
      unless time =~ /^\d\d:\d\d:\d\d$/
        logger.error "Time #{time} invalid on link #{link_text}"
        time = nil
      end
      parse_sub_day_speech_page(sub_page, time, debates, date, house)
    #elsif link_text =~ /^Procedural text:/
    #  # Assuming no time recorded for Procedural text
    #  parse_sub_day_speech_page(sub_page, nil, debates, date)
    elsif link_text == "Official Hansard" || link_text =~ /^Start of Business/ || link_text == "Adjournment"
      # Do nothing - skip this entirely
    elsif link_text =~ /^Procedural text:/ || link_text =~ /^QUESTIONS IN WRITING:/ || link_text =~ /^Division:/ ||
        link_text =~ /^REQUESTS? FOR DETAILED INFORMATION:/ ||
        link_text =~ /^Petition:/ || link_text =~ /^PRIVILEGE:/ || link_text == "Interruption" ||
        link_text =~ /^QUESTIONS? ON NOTICE:/i || link_text =~ /^QUESTIONS TO THE SPEAKER/
      #logger.info "Not yet supporting: #{link_text}"
    # Hack to deal with incorrectly titled page on 31 Oct 2005 
    elsif link_text =~ /^IRAQ/
      #logger.info "Not yet supporting: #{link_text}"
    else
      throw "Unsupported: #{link_text}"
    end
  end

  def parse_sub_day_speech_page(sub_page, time, debates, date, house)
    top_content_tag = sub_page.search('div#contentstart').first
    throw "Page on date #{date} at time #{time} has no content" if top_content_tag.nil?
    
    # Extract permanent URL of this subpage. Also, quoting because there is a bug
    # in XML Builder that for some reason is not quoting attributes properly
    url = quote(sub_page.links.text("[Permalink]").uri.to_s)

    newtitle = sub_page.search('div#contentstart div.hansardtitle').map { |m| m.inner_html }.join('; ')
    newsubtitle = sub_page.search('div#contentstart div.hansardsubtitle').map { |m| m.inner_html }.join('; ')
    # Replace any unicode characters
    newtitle = replace_unicode(newtitle)
    newsubtitle = replace_unicode(newsubtitle)

    debates.add_heading(newtitle, newsubtitle, url)

    speaker = nil
    top_content_tag.children.each do |e|
      class_value = e.attributes["class"]
      if e.name == "div"
        if class_value == "hansardtitlegroup" || class_value == "hansardsubtitlegroup"
        elsif class_value == "speech0" || class_value == "speech1"
          e.children[1..-1].each do |e|
            speaker = parse_speech_block(e, speaker, time, url, debates, date, house)
            debates.increment_minor_count
          end
        elsif class_value == "motionnospeech" || class_value == "subspeech0" || class_value == "subspeech1" ||
            class_value == "motion" || class_value = "quote"
          speaker = parse_speech_block(e, speaker, time, url, debates, date, house)
          debates.increment_minor_count
        else
          throw "Unexpected class value #{class_value} for tag #{e.name}"
        end
      elsif e.name == "p"
        speaker = parse_speech_block(e, speaker, time, url, debates, date, house)
        debates.increment_minor_count
      elsif e.name == "table"
        if class_value == "division"
          debates.increment_minor_count
          # Ignore (for the time being)
        else
          throw "Unexpected class value #{class_value} for tag #{e.name}"
        end
      else
        throw "Unexpected tag #{e.name}"
      end
    end
  end<commit_msg>Handle situation where child can be a text node<commit_after>require 'speech'
require 'mechanize_proxy'
require 'configuration'
require 'debates'
require 'builder_alpha_attributes'
require 'house'
require 'people_image_downloader'

class UnknownSpeaker
  def initialize(name)
    @name = name
  end
  
  def id
    "unknown"
  end
  
  def name
    Name.title_first_last(@name)
  end
end

require 'rubygems'
require 'log4r'

class HansardParser
  attr_reader :logger
  
  def initialize(people)
    @people = people
    conf = Configuration.new
    
    # Set up logging
    @logger = Log4r::Logger.new 'HansardParser'
    # Log to both standard out and the file set in configuration.yml
    @logger.add(Log4r::Outputter.stdout)
    @logger.add(Log4r::FileOutputter.new('foo', :filename => conf.log_path, :trunc => false,
      :formatter => Log4r::PatternFormatter.new(:pattern => "[%l] %d :: %M")))

    bio = PeopleImageDownloader.new
    bio.attach_aph_person_ids(people)
  end
  
  def parse_date_house(date, xml_filename, house)
    @logger.info "Parsing #{house} speeches for #{date.strftime('%a %d %b %Y')}..."
    url = "http://parlinfoweb.aph.gov.au/piweb/browse.aspx?path=Chamber%20%3E%20#{house.representatives? ? "House" : "Senate"}%20Hansard%20%3E%20#{date.year}%20%3E%20#{date.day}%20#{Date::MONTHNAMES[date.month]}%20#{date.year}"

    debates = Debates.new(date, house)
    
    # Required to workaround long viewstates generated by .NET (whatever that means)
    # See http://code.whytheluckystiff.net/hpricot/ticket/13
    Hpricot.buffer_size = 1600000

    agent = MechanizeProxy.new
    agent.cache_subdirectory = date.to_s

    begin
      page = agent.get(url)
      # HACK: Don't know why if the page isn't found a return code isn't returned. So, hacking around this.
      if page.title == "ParlInfo Web - Error"
        throw "ParlInfo Web - Error"
      end
    rescue
      logger.warn "Could not retrieve overview page for date #{date}"
      return
    end
    # Structure of the page is such that we are only interested in some of the links
    page.links[30..-4].each do |link|
      begin
        parse_sub_day_page(link.to_s, agent.click(link), debates, date, house)
      rescue
        logger.error "Exception thrown during processing of sub page: #{link}"
        raise $!
      end
      # This ensures that every sub day page has a different major count which limits the impact
      # of when we start supporting things like written questions, procedurial text, etc..
      debates.increment_major_count
    end
    
    debates.output(xml_filename)
  end
  
  def parse_sub_day_page(link_text, sub_page, debates, date, house)
    # Only going to consider speeches for the time being
    if link_text =~ /^Speech:/ || link_text =~ /^QUESTIONS? WITHOUT NOTICE/i || link_text =~ /^QUESTIONS TO THE SPEAKER:/
      # Link text for speech has format:
      # HEADING > NAME > HOUR:MINS:SECS
      split = link_text.split('>').map{|a| a.strip}
      logger.error "Expected split to have length 3 in link text: #{link_text}" unless split.size == 3
      time = split[2]
      # Check that time is something valid
      unless time =~ /^\d\d:\d\d:\d\d$/
        logger.error "Time #{time} invalid on link #{link_text}"
        time = nil
      end
      parse_sub_day_speech_page(sub_page, time, debates, date, house)
    #elsif link_text =~ /^Procedural text:/
    #  # Assuming no time recorded for Procedural text
    #  parse_sub_day_speech_page(sub_page, nil, debates, date)
    elsif link_text == "Official Hansard" || link_text =~ /^Start of Business/ || link_text == "Adjournment"
      # Do nothing - skip this entirely
    elsif link_text =~ /^Procedural text:/ || link_text =~ /^QUESTIONS IN WRITING:/ || link_text =~ /^Division:/ ||
        link_text =~ /^REQUESTS? FOR DETAILED INFORMATION:/ ||
        link_text =~ /^Petition:/ || link_text =~ /^PRIVILEGE:/ || link_text == "Interruption" ||
        link_text =~ /^QUESTIONS? ON NOTICE:/i || link_text =~ /^QUESTIONS TO THE SPEAKER/
      #logger.info "Not yet supporting: #{link_text}"
    # Hack to deal with incorrectly titled page on 31 Oct 2005 
    elsif link_text =~ /^IRAQ/
      #logger.info "Not yet supporting: #{link_text}"
    else
      throw "Unsupported: #{link_text}"
    end
  end

  def parse_sub_day_speech_page(sub_page, time, debates, date, house)
    top_content_tag = sub_page.search('div#contentstart').first
    throw "Page on date #{date} at time #{time} has no content" if top_content_tag.nil?
    
    # Extract permanent URL of this subpage. Also, quoting because there is a bug
    # in XML Builder that for some reason is not quoting attributes properly
    url = quote(sub_page.links.text("[Permalink]").uri.to_s)

    newtitle = sub_page.search('div#contentstart div.hansardtitle').map { |m| m.inner_html }.join('; ')
    newsubtitle = sub_page.search('div#contentstart div.hansardsubtitle').map { |m| m.inner_html }.join('; ')
    # Replace any unicode characters
    newtitle = replace_unicode(newtitle)
    newsubtitle = replace_unicode(newsubtitle)

    debates.add_heading(newtitle, newsubtitle, url)

    speaker = nil
    top_content_tag.children.each do |e|
      break unless e.respond_to?(:attributes)
      
      class_value = e.attributes["class"]
      if e.name == "div"
        if class_value == "hansardtitlegroup" || class_value == "hansardsubtitlegroup"
        elsif class_value == "speech0" || class_value == "speech1"
          e.children[1..-1].each do |e|
            speaker = parse_speech_block(e, speaker, time, url, debates, date, house)
            debates.increment_minor_count
          end
        elsif class_value == "motionnospeech" || class_value == "subspeech0" || class_value == "subspeech1" ||
            class_value == "motion" || class_value = "quote"
          speaker = parse_speech_block(e, speaker, time, url, debates, date, house)
          debates.increment_minor_count
        else
          throw "Unexpected class value #{class_value} for tag #{e.name}"
        end
      elsif e.name == "p"
        speaker = parse_speech_block(e, speaker, time, url, debates, date, house)
        debates.increment_minor_count
      elsif e.name == "table"
        if class_value == "division"
          debates.increment_minor_count
          # Ignore (for the time being)
        else
          throw "Unexpected class value #{class_value} for tag #{e.name}"
        end
      else
        throw "Unexpected tag #{e.name}"
      end
    end
  end
{'fraction_non_alphanumeric': 0.1141157382368848, 'fraction_numerical': 0.008189755079966004, 'mean_word_length': 3.052598622417032, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '314792', 'n_tokens_mistral': 4318, 'n_tokens_neox': 4089, 'n_words': 1154}