<commit_before>module DataLoader
  class DivisionXML
    def initialize(division_xml, house)
      @division_xml = division_xml
      @house = House.australian_to_uk(house)
    end

    def date
      @division_xml.attr(:divdate)
    end

    def number
      @division_xml.attr(:divnumber)
    end

    def house
      @house
    end

    def name
      text = if !major_heading.blank? && !minor_heading.blank?
        major_heading + ' &#8212; ' + minor_heading
      elsif !major_heading.blank?
        major_heading
      elsif !minor_heading.blank?
        minor_heading
      end
      title_case(text).gsub('—', ' &#8212; ')
    end

    def source_url
      @division_xml.attr(:url)
    end

    def debate_url
      # TODO: PHP always gets the previous heading, major or minor. Is this to support missing headings?
      preceeding_minor_heading_element.attr(:url)
    end

    def source_gid
      @division_xml.attr(:id)
    end

    def debate_gid
      # TODO: PHP always gets the previous heading, major or minor. Is this to support missing headings?
      preceeding_minor_heading_element.attr(:id)
    end

    def motion
      pwmotiontext = pwmotiontexts.map { |p| p.to_s + "\n\n" }.join
      text = pwmotiontext.empty? ? previous_speeches.map { |s| speech_text s }.join : pwmotiontext
      # Truncate really long motion text at the same size as formatted_motion_text
      Rails.logger.warn "Truncating very long motion text for division: #{division.house} #{division.date} #{division.number}" if text.size > 15000
      encode_html_entities(ActionController::Base.helpers.truncate(text, length: 15000))
    end

    def clock_time
      time = @division_xml.attr(:time)
      time = "#{time}:00" if time =~ /^\d\d:\d\d$/
      time = "0#{time}" if time =~ /^\d\d:\d\d:\d\d$/

      if time !~ /^\d\d\d:\d\d:\d\d$/
        Rails.logger.warn "Clock time '#{time}' not in right format"
        ''
      else<commit_msg>Oops, didn't commit copy pasta code :(<commit_after>module DataLoader
  class DivisionXML
    def initialize(division_xml, house)
      @division_xml = division_xml
      @house = House.australian_to_uk(house)
    end

    def date
      @division_xml.attr(:divdate)
    end

    def number
      @division_xml.attr(:divnumber)
    end

    def house
      @house
    end

    def name
      text = if !major_heading.blank? && !minor_heading.blank?
        major_heading + ' &#8212; ' + minor_heading
      elsif !major_heading.blank?
        major_heading
      elsif !minor_heading.blank?
        minor_heading
      end
      title_case(text).gsub('—', ' &#8212; ')
    end

    def source_url
      @division_xml.attr(:url)
    end

    def debate_url
      # TODO: PHP always gets the previous heading, major or minor. Is this to support missing headings?
      preceeding_minor_heading_element.attr(:url)
    end

    def source_gid
      @division_xml.attr(:id)
    end

    def debate_gid
      # TODO: PHP always gets the previous heading, major or minor. Is this to support missing headings?
      preceeding_minor_heading_element.attr(:id)
    end

    def motion
      pwmotiontext = pwmotiontexts.map { |p| p.to_s + "\n\n" }.join
      text = pwmotiontext.empty? ? previous_speeches.map { |s| speech_text s }.join : pwmotiontext
      # Truncate really long motion text at the same size as formatted_motion_text
      Rails.logger.warn "Truncating very long motion text for division: #{house} #{date} #{number}" if text.size > 15000
      encode_html_entities(ActionController::Base.helpers.truncate(text, length: 15000))
    end

    def clock_time
      time = @division_xml.attr(:time)
      time = "#{time}:00" if time =~ /^\d\d:\d\d$/
      time = "0#{time}" if time =~ /^\d\d:\d\d:\d\d$/

      if time !~ /^\d\d\d:\d\d:\d\d$/
        Rails.logger.warn "Clock time '#{time}' not in right format"
        ''
      else
{'fraction_non_alphanumeric': 0.10970573051109964, 'fraction_numerical': 0.010841507485802787, 'mean_word_length': 2.587962962962963, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 16}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '102175', 'n_tokens_mistral': 1334, 'n_tokens_neox': 1241, 'n_words': 329}