<commit_before>module WebPageParser
  class RtePageParserFactory < WebPageParser::ParserFactory
    URL_RE = ORegexp.new("(www\.)?rte\.ie/[a-z-]+/[0-9]{4}/[0-9]{4}/[a-z-]+.[a-z-]{4}")
    INVALID_URL_RE = ORegexp.new("/sport/")

    def self.can_parse?(options)
      url = options[:url].split('#').first
      return nil if INVALID_URL_RE.match(url)
      URL_RE.match(url)
    end
    
    def self.create(options = {})
      RtePageParserV1.new(options)
    end
  end

  class RtePageParserV1 < WebPageParser::BaseParser
    ICONV = nil
    TITLE_RE = ORegexp.new('<meta name="DC.title" scheme="DCTERMS.URI" content="(.*)"', 'i')
    DATE_RE = ORegexp.new('<meta name="datemodified" content="(.*)"', 'i')
    DESC_RE = ORegexp.new('<meta name="description" content="(.*)" />')
    CONTENT_RE = ORegexp.new('itemprop="articleBody"(.*)<div id="user-options-bottom">', 'm')
    STRIP_TAGS_RE = ORegexp.new('</?(strong|br|a|span|div|img|tr|td|!--|table)[^>]*>','i')
    PARA_RE = Regexp.new(/<(p|h2)[^>]*>(.*?)<\/\1>/i)
    KILL_CHARS_RE = ORegexp.new('[\r\n]+')

    private
    
    def date_processor
      begin
        # OPD is in GMT/UTC, which DateTime seems to use by default
        @date = DateTime.parse(@date)
      rescue ArgumentError
        @date = Time.now.utc
      end
    end

    def content_processor
      @content = STRIP_TAGS_RE.gsub(@content, '')
      @content = @content.scan(PARA_RE).collect { |a| a[1] }

      # Content processing doesn't remove "&nbsp;", done
      @content = @content.map {|p| p.gsub('&nbsp;', " ")}
    end
    
  end
end<commit_msg>Adjusted RTE URL matcher for new style URLs<commit_after>module WebPageParser
  class RtePageParserFactory < WebPageParser::ParserFactory
    URL_RE = ORegexp.new("(www\.)?rte\.ie\/(?:[a-z-]+\/)+[0-9]{4}\/[0-9]{4}\/[0-9a-z-]+.[a-z-]\/")
    INVALID_URL_RE = ORegexp.new("/sport/")

    def self.can_parse?(options)
      url = options[:url].split('#').first
      return nil if INVALID_URL_RE.match(url)
      URL_RE.match(url)
    end
    
    def self.create(options = {})
      RtePageParserV1.new(options)
    end
  end

  class RtePageParserV1 < WebPageParser::BaseParser
    ICONV = nil
    TITLE_RE = ORegexp.new('<meta name="DC.title" scheme="DCTERMS.URI" content="(.*)"', 'i')
    DATE_RE = ORegexp.new('<meta name="datemodified" content="(.*)"', 'i')
    DESC_RE = ORegexp.new('<meta name="description" content="(.*)" />')
    CONTENT_RE = ORegexp.new('itemprop="articleBody"(.*)<div id="user-options-bottom">', 'm')
    STRIP_TAGS_RE = ORegexp.new('</?(iframe|strong|br|a|span|div|img|tr|td|!--|table)[^>]*>','i')
    PARA_RE = Regexp.new(/<(p|h2)[^>]*>(.*?)<\/\1>/i)
    KILL_CHARS_RE = ORegexp.new('[\r\n]+')

    private
    
    def date_processor
      begin
        # OPD is in GMT/UTC, which DateTime seems to use by default
        @date = DateTime.parse(@date)
      rescue ArgumentError
        @date = Time.now.utc
      end
    end

    def content_processor
      @content = STRIP_TAGS_RE.gsub(@content, '')
      @content = @content.scan(PARA_RE).collect { |a| a[1] }

      # Content processing doesn't remove "&nbsp;", done
      @content = @content.map {|p| p.gsub('&nbsp;', " ")}
    end
    
  end
end
{'fraction_non_alphanumeric': 0.19508400746733043, 'fraction_numerical': 0.007778469197261979, 'mean_word_length': 3.11651728553137, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '245248', 'n_tokens_mistral': 1252, 'n_tokens_neox': 1220, 'n_words': 202}