<commit_before># coding: utf-8
module Disher
  class PunctuationLogic < Logic
    @@default_options = {
      :punctuations => %w(、 。 ！ ？),
      :extract_tags => %w(div td section article),
      :plus_words => [/main/],
      :minus_words => [/comment/, /side/, /menu/],
      :punctuations_weight => 1.8,
      :children_weight => -0.5,
      :links_weight => -1,
      :plus_words_weight => 2,
      :minus_words_weight => -1.5
    }

    def extract(body)
      doc = Nokogiri::HTML(
        body.encode("UTF-8",
          :invalid => :replace, :undef => :replace, :replace => ""))
      survey(doc)
      elements = []
      @options[:extract_tags].each do |tag|
        elements << doc.xpath("//#{tag}")
      end
      score = 0
      content = ''
      elements.flatten.each do |elm|
        c = calculate_score(elm)
        if c > score
          content = elm.inner_html
          score = c
        end
      end
      content
    end

    private
    def survey(doc)
      @punctuations_count = doc.text.count(@options[:punctuations].join)
      @children_count     = doc.xpath(".//*").count
      @links_count        = doc.xpath(".//a['href']").count
      @plus_words_count   = @options[:plus_words].inject(0) {|count, word| count += doc.text.scan(word).count }
      @minus_words_count  = @options[:minus_words].inject(0) {|count, word| count += doc.text.scan(word).count }
    end

    def calculate_score(element)
      punctuations_rate = element.text.count(@options[:punctuations].join).to_f / @punctuations_count
      children_rate     = element.xpath(".//*").count.to_f / @children_count
      links_rate        = element.xpath(".//a['href']").count.to_f / @links_count
      plus_words_rate   = @options[:plus_words].inject(0) {|count, word| count += element.text.scan(word).count }.to_f / @plus_words_count
      minus_words_rate  = @options[:minus_words].inject(0) {|count, word| count += element.text.scan(word).count }.to_f / @minus_words_count

      score = punctuations_rate * @options[:punctuations_weight] +
        children_rate * @options[:children_weight] +
        links_rate * @options[:links_weight] +
        plus_words_rate * @options[:plus_words_weight] +
        minus_words_rate * @options[:minus_words_weight]

      if @options[:debug]
        puts element.inspect
        puts "score: #{score}"
        puts "punctuations: #{punctuations_rate}"
        puts "children: #{children_rate}"
        puts "links: #{links_rate}"
        puts "plus_words: #{plus_words_rate}"
        puts "minus_words: #{minus_words_rate}"
        puts "=========================================="
      end

      score
    end
  end<commit_msg>Prevent rate to be NaN<commit_after># coding: utf-8
module Disher
  class PunctuationLogic < Logic
    @@default_options = {
      :punctuations => %w(、 。 ！ ？),
      :extract_tags => %w(div td section article),
      :plus_words => [/main/],
      :minus_words => [/comment/, /side/, /menu/],
      :punctuations_weight => 1.8,
      :children_weight => -0.5,
      :links_weight => -1,
      :plus_words_weight => 2,
      :minus_words_weight => -1.5
    }

    def extract(body)
      doc = Nokogiri::HTML(
        body.encode("UTF-8",
          :invalid => :replace, :undef => :replace, :replace => ""))
      survey(doc)
      elements = []
      @options[:extract_tags].each do |tag|
        elements << doc.xpath("//#{tag}")
      end
      score = 0
      content = ''
      elements.flatten.each do |elm|
        c = calculate_score(elm)
        if c > score
          content = elm.inner_html
          score = c
        end
      end
      content
    end

    private
    def survey(doc)
      @punctuations_count = doc.text.count(@options[:punctuations].join)
      @children_count     = doc.xpath(".//*").count
      @links_count        = doc.xpath(".//a['href']").count
      @plus_words_count   = @options[:plus_words].inject(0) {|count, word| count += doc.text.scan(word).count }
      @minus_words_count  = @options[:minus_words].inject(0) {|count, word| count += doc.text.scan(word).count }
    end

    def calculate_score(element)
      punctuations_rate = element.text.count(@options[:punctuations].join).to_f / @punctuations_count
      children_rate     = element.xpath(".//*").count.to_f / @children_count
      links_rate        = element.xpath(".//a['href']").count.to_f / @links_count
      plus_words_rate   = @options[:plus_words].inject(0) {|count, word| count += element.text.scan(word).count }.to_f / @plus_words_count
      minus_words_rate  = @options[:minus_words].inject(0) {|count, word| count += element.text.scan(word).count }.to_f / @minus_words_count

      punctuations_rate = 0 if punctuations_rate.nan?
      children_rate     = 0 if children_rate.nan?
      links_rate        = 0 if links_rate.nan?
      plus_words_rate   = 0 if plus_words_rate.nan?
      minus_words_rate  = 0 if minus_words_rate.nan?

      score = punctuations_rate * @options[:punctuations_weight] +
        children_rate * @options[:children_weight] +
        links_rate * @options[:links_weight] +
        plus_words_rate * @options[:plus_words_weight] +
        minus_words_rate * @options[:minus_words_weight]

      if @options[:debug]
        puts element.inspect
        puts "score: #{score}"
        puts "punctuations: #{punctuations_rate}"
        puts "children: #{children_rate}"
        puts "links: #{links_rate}"
        puts "plus_words: #{plus_words_rate}"
        puts "minus_words: #{minus_words_rate}"
        puts "=========================================="
      end

      score
    end
  end
{'fraction_non_alphanumeric': 0.15756602426837973, 'fraction_numerical': 0.006245538900785153, 'mean_word_length': 2.6254851228978007, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1065964', 'n_tokens_mistral': 1825, 'n_tokens_neox': 1810, 'n_words': 337}