<commit_before>require 'link_thumbnailer/scrapers/default/base'

module LinkThumbnailer
  module Scrapers
    module Default
      class WordsCount < ::LinkThumbnailer::Scrapers::Default::Base

        def value
          return count_from_body
          nil
        end

        private

        def count_from_body
          nodes_from_body.map{|a| a.text}.join(' ').split.reject { |w| w !~ /([a-zA-Z]+(_[a-zA-Z]+)*)/ }.count
        end

        def node_from_meta
          @node_from_meta ||= meta_xpath(key: :name)
        end

        def nodes_from_body
          candidates.select { |node| valid_paragraph?(node) }
        end

        def valid_paragraph?(node)
          true
        end

        def candidates
          document.css('p,td,h1,h2,h3,h4,h5,h6,span')
        end

        def modelize(node, text, i = 1)
          model_class.new(node, text, i)
        end

      end<commit_msg>encode and replace not correct chars<commit_after>require 'link_thumbnailer/scrapers/default/base'

module LinkThumbnailer
  module Scrapers
    module Default
      class WordsCount < ::LinkThumbnailer::Scrapers::Default::Base

        def value
          return count_from_body
          nil
        end

        private

        def count_from_body
          nodes_from_body.map{|a| a.text}.join(' ').encode('UTF-8', :invalid => :replace).split.reject { |w| w !~ /([a-zA-Z]+(_[a-zA-Z]+)*)/ }.count
        end

        def node_from_meta
          @node_from_meta ||= meta_xpath(key: :name)
        end

        def nodes_from_body
          candidates.select { |node| valid_paragraph?(node) }
        end

        def valid_paragraph?(node)
          true
        end

        def candidates
          document.css('p,td,h1,h2,h3,h4,h5,h6,span')
        end

        def modelize(node, text, i = 1)
          model_class.new(node, text, i)
        end

      end
{'fraction_non_alphanumeric': 0.10849812934259755, 'fraction_numerical': 0.008017103153393906, 'mean_word_length': 1.9667194928684628, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6535806', 'n_tokens_mistral': 627, 'n_tokens_neox': 601, 'n_words': 122}