<commit_before>require 'speech'
require 'mechanize_proxy'
require 'configuration'
require 'debates'

class UnknownSpeaker
  def initialize(name)
    @name = name
  end
  
  def id
    "unknown"
  end
  
  def name
    Name.title_first_last(@name)
  end
end

require 'rubygems'
require 'log4r'

class HansardParser
  attr_reader :logger
  
  def initialize(people)
    @people = people
    conf = Configuration.new
    
    # Set up logging
    @logger = Log4r::Logger.new 'HansardParser'
    # Log to both standard out and the file set in configuration.yml
    @logger.add(Log4r::Outputter.stdout)
    @logger.add(Log4r::FileOutputter.new('foo', :filename => conf.log_path, :trunc => false,
      :formatter => Log4r::PatternFormatter.new(:pattern => "[%l] %d :: %M")))
  end
  
  def parse_date(date, xml_filename)
    debates = Debates.new(date)
    
    @logger.info "Parsing speeches for #{date.strftime('%a %d %b %Y')}..."
    
    # Required to workaround long viewstates generated by .NET (whatever that means)
    # See http://code.whytheluckystiff.net/hpricot/ticket/13
    Hpricot.buffer_size = 400000

    agent = MechanizeProxy.new
    agent.cache_subdirectory = date.to_s

    url = "http://parlinfoweb.aph.gov.au/piweb/browse.aspx?path=Chamber%20%3E%20House%20Hansard%20%3E%20#{date.year}%20%3E%20#{date.day}%20#{Date::MONTHNAMES[date.month]}%20#{date.year}"
    begin
      page = agent.get(url)
      # HACK: Don't know why if the page isn't found a return code isn't returned. So, hacking around this.
      if page.title == "ParlInfo Web - Error"
        throw "ParlInfo Web - Error"
      end
    rescue
      logger.warn "Could not retrieve overview page for date #{date}"
      return
    end
    # Structure of the page is such that we are only interested in some of the links
    page.links[30..-4].each do |link|
      parse_sub_day_page(link.to_s, agent.click(link), debates, date)
      # This ensures that every sub day page has a different major count which limits the impact
      # of when we start supporting things like written questions, procedurial text, etc..
      debates.increment_major_count
    end
    
    debates.output(xml_filename)
  end
  
  def parse_sub_day_page(link_text, sub_page, debates, date)
    # Only going to consider speeches for the time being
    if link_text =~ /^Speech:/
      # Link text for speech has format:
      # HEADING > NAME > HOUR:MINS:SECS
      split = link_text.split('>').map{|a| a.strip}
      logger.error "Expected split to have length 3" unless split.size == 3
      time = split[2]
      # Extract permanent URL of this subpage. Also, quoting because there is a bug
      # in XML Builder that for some reason is not quoting attributes properly
      url = quote(sub_page.links.text("[Permalink]").uri.to_s)
      
      parse_sub_day_speech_page(sub_page, time, url, debates, date)
    elsif link_text == "Official Hansard" || link_text =~ /^Start of Business/ || link_text == "Adjournment"
      # Do nothing - skip this entirely
    elsif link_text =~ /^Procedural text:/ || link_text =~ /^QUESTIONS WITHOUT NOTICE:/ || link_text =~ /^QUESTIONS IN WRITING:/ ||
      link_text =~ /^Division:/
      logger.warn "Not yet supporting: #{link_text}"
    else
      logger.warn "Unsupported: #{link_text}"
    end
  end

  def parse_sub_day_speech_page(sub_page, time, url, debates, date)
    newtitle = sub_page.search('div#contentstart div.hansardtitle').map { |m| m.inner_html }.join('; ')
    newsubtitle = sub_page.search('div#contentstart div.hansardsubtitle').map { |m| m.inner_html }.join('; ')
    # Replace any unicode characters
    newtitle = replace_unicode(newtitle)
    newsubtitle = replace_unicode(newsubtitle)

    debates.add_heading(newtitle, newsubtitle, url)

    content = sub_page.search('div#contentstart > div.speech0 > *')

    if content.size > 1
      speaker = nil
      content[1..-1].each do |e|
        speakername = extract_speakername(e)
        # Only change speaker if a speaker name was found
        speaker = lookup_speaker(speakername, date) if speakername
        debates.add_speech(speaker, time, url, clean_speech_content(url, e))
      end
    else
      # Doing this just to ensure that regression tests don't fail
      #cleaned = clean_speech_content(url, content[0])
      #p cleaned
      debates.add_speech(nil, time, url, Hpricot(''))
    end
  end
  
  # Pass in current speaker (if there is one)
  def parse_speeches(content, speaker = nil)
    content.each do |e|
      speakername = extract_speakername(e)
      # Only change speaker if a speaker name was found
      speaker = lookup_speaker(speakername, date) if speakername
      debates.add_speech(speaker, time, url, clean_speech_content(url, e))<commit_msg>url is now captured inside parse_sub_day_speech_page<commit_after>require 'speech'
require 'mechanize_proxy'
require 'configuration'
require 'debates'

class UnknownSpeaker
  def initialize(name)
    @name = name
  end
  
  def id
    "unknown"
  end
  
  def name
    Name.title_first_last(@name)
  end
end

require 'rubygems'
require 'log4r'

class HansardParser
  attr_reader :logger
  
  def initialize(people)
    @people = people
    conf = Configuration.new
    
    # Set up logging
    @logger = Log4r::Logger.new 'HansardParser'
    # Log to both standard out and the file set in configuration.yml
    @logger.add(Log4r::Outputter.stdout)
    @logger.add(Log4r::FileOutputter.new('foo', :filename => conf.log_path, :trunc => false,
      :formatter => Log4r::PatternFormatter.new(:pattern => "[%l] %d :: %M")))
  end
  
  def parse_date(date, xml_filename)
    debates = Debates.new(date)
    
    @logger.info "Parsing speeches for #{date.strftime('%a %d %b %Y')}..."
    
    # Required to workaround long viewstates generated by .NET (whatever that means)
    # See http://code.whytheluckystiff.net/hpricot/ticket/13
    Hpricot.buffer_size = 400000

    agent = MechanizeProxy.new
    agent.cache_subdirectory = date.to_s

    url = "http://parlinfoweb.aph.gov.au/piweb/browse.aspx?path=Chamber%20%3E%20House%20Hansard%20%3E%20#{date.year}%20%3E%20#{date.day}%20#{Date::MONTHNAMES[date.month]}%20#{date.year}"
    begin
      page = agent.get(url)
      # HACK: Don't know why if the page isn't found a return code isn't returned. So, hacking around this.
      if page.title == "ParlInfo Web - Error"
        throw "ParlInfo Web - Error"
      end
    rescue
      logger.warn "Could not retrieve overview page for date #{date}"
      return
    end
    # Structure of the page is such that we are only interested in some of the links
    page.links[30..-4].each do |link|
      parse_sub_day_page(link.to_s, agent.click(link), debates, date)
      # This ensures that every sub day page has a different major count which limits the impact
      # of when we start supporting things like written questions, procedurial text, etc..
      debates.increment_major_count
    end
    
    debates.output(xml_filename)
  end
  
  def parse_sub_day_page(link_text, sub_page, debates, date)
    # Only going to consider speeches for the time being
    if link_text =~ /^Speech:/
      # Link text for speech has format:
      # HEADING > NAME > HOUR:MINS:SECS
      split = link_text.split('>').map{|a| a.strip}
      logger.error "Expected split to have length 3" unless split.size == 3
      time = split[2]
      
      parse_sub_day_speech_page(sub_page, time, debates, date)
    elsif link_text == "Official Hansard" || link_text =~ /^Start of Business/ || link_text == "Adjournment"
      # Do nothing - skip this entirely
    elsif link_text =~ /^Procedural text:/ || link_text =~ /^QUESTIONS WITHOUT NOTICE:/ || link_text =~ /^QUESTIONS IN WRITING:/ ||
      link_text =~ /^Division:/
      logger.warn "Not yet supporting: #{link_text}"
    else
      logger.warn "Unsupported: #{link_text}"
    end
  end

  def parse_sub_day_speech_page(sub_page, time, debates, date)
    # Extract permanent URL of this subpage. Also, quoting because there is a bug
    # in XML Builder that for some reason is not quoting attributes properly
    url = quote(sub_page.links.text("[Permalink]").uri.to_s)

    newtitle = sub_page.search('div#contentstart div.hansardtitle').map { |m| m.inner_html }.join('; ')
    newsubtitle = sub_page.search('div#contentstart div.hansardsubtitle').map { |m| m.inner_html }.join('; ')
    # Replace any unicode characters
    newtitle = replace_unicode(newtitle)
    newsubtitle = replace_unicode(newsubtitle)

    debates.add_heading(newtitle, newsubtitle, url)

    content = sub_page.search('div#contentstart > div.speech0 > *')

    if content.size > 1
      speaker = nil
      content[1..-1].each do |e|
        speakername = extract_speakername(e)
        # Only change speaker if a speaker name was found
        speaker = lookup_speaker(speakername, date) if speakername
        debates.add_speech(speaker, time, url, clean_speech_content(url, e))
      end
    else
      # Doing this just to ensure that regression tests don't fail
      debates.add_speech(nil, time, url, Hpricot(''))
    end
  end
  
  # Pass in current speaker (if there is one)
  def parse_speeches(content, speaker = nil)
    content.each do |e|
      speakername = extract_speakername(e)
      # Only change speaker if a speaker name was found
      speaker = lookup_speaker(speakername, date) if speakername
      debates.add_speech(speaker, time, url, clean_speech_content(url, e))
{'fraction_non_alphanumeric': 0.10662153585783796, 'fraction_numerical': 0.00941400465411466, 'mean_word_length': 3.260928346101848, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '473596', 'n_tokens_mistral': 3180, 'n_tokens_neox': 3022, 'n_words': 861}