<commit_before># encoding: utf-8

require 'wagg/crawler/vote'

module Wagg
  module Crawler
    class Comment
      attr_reader :id, :author, :timestamps
      attr_reader :news_url, :news_index
      attr_accessor :votes_count, :karma

      def initialize(id, author, body, timestamps, news_url, news_index)
        @id = id
        @author = author
        @body = body
        @timestamps = timestamps
        @news_url = news_url
        @news_index = news_index

        @karma = nil
        @votes_count = nil

        @votes = nil

        @votes_closed = (@timestamps['creation'] + Wagg::Utils::Constants::COMMENT_VOTES_LIFETIME) <= @timestamps['retrieval']
      end

      def body
        @body.to_s.scrub.strip
      end

      def votes
        if @votes.nil? && self.votes_available?
          # TODO Revise the karma.nil and votes_count.nil case
          #if !@karma.nil? && !@votes_count.nil?
            @votes = Vote.parse_comment_votes(@id)
          #end
        end

        @votes
      end

      def voting_closed?
        @votes_closed
      end

      def voting_open?
        !@votes_closed
      end

      def modified?
        @timestamps.has_key?('edition') && !@timestamps['edition'].nil? && @timestamps['edition'] > 0
      end

      def votes?
        # TODO Revise the karma.nil and votes_count.nil case
        #self.votes_available? && @votes.size > 0
        self.votes_available? && @votes_count > 0
      end

      def votes_available?
        (@timestamps['creation'] <= @timestamps['retrieval']) &&
            (@timestamps['retrieval'] <= (@timestamps['creation'] + Wagg::Utils::Constants::COMMENT_VOTES_LIFETIME + Wagg::Utils::Constants::COMMENT_CONTRIBUTION_LIFETIME))
      end

      def votes_consistent?
        self.votes? ? @votes_count == self.votes.size : FALSE
      end

      def parent_index
        referred_comments_list = @body.search('.//a[contains(concat(" ", normalize-space(@class), " "), " tooltip ")]')
        @body.each { |x| puts x.class }
        puts referred_comments_list
        #referred_comments_list = @body.search('//a[matches(@class, "tooltip c:%{cid}-\d+"]' % {cid: @id})
        parent_comment_item = referred_comments_list.first
        parent_comment_index = Wagg::Utils::Functions.str_at_xpath(@body, './a/text()').to_i

        parent_comment_index
      end

      def position_in_news
        Wagg::Utils::Functions.str_at_xpath(@body, './a/strong/text()')[/(?<position>\d+)/].to_i
      end

      def to_s
        "COMMENT : %{id} - %{a}" % {id:@id, a:@author} +
            "\n" +
            "    %{news_index} - %{news_url}" % {news_index:@news_index, news_url:@news_url} +
            "\n" +
            "    %{ts}" % {ts:@timestamps} +
            "\n" +
            "    %{b}..." % {b:@body[0,20]} +
            "\n" +
            "    %{k} :: %{vc}" % {k:@karma, vc:@votes_count} +
            "\n" +
            "    (%{vc}) => %{v}" % {vc:(@votes.nil? ? 'EMPTY' : @votes.size), v:@votes}
      end

      class << self
        def parse(item, retrieval_timestamp=Time.now.to_i)
          # Parse comment's body data
          body_item = item.search('.//div[contains(concat(" ", normalize-space(@class), " "), " comment-body ")]')
          #comment_body = body_item.search('./child::node()').to_s.scrub.strip
          comment_body = body_item.search('./child::node()')
          comment_id = Wagg::Utils::Functions.str_at_xpath(body_item, './@id')[/(?!c-)(?<id>\d+)/].to_i
          # Also available at unique id: //*[@id="cid-XXXXXXXX"]/a/@href
          # TODO: Use regex to remove last element in extraced href instead of these functions...
          comment_news_url = Wagg::Utils::Constants::SITE_URL + Wagg::Utils::Functions.str_at_xpath(body_item, './a/@href').split('/')[0...-1].join('/')
          comment_news_index = Wagg::Utils::Functions.str_at_xpath(item, './@id')[/(?!cid-)(?<id>\d+)/].to_i

          # Parse comment's authorship meta data
          meta_item = item.search('.//div[contains(concat(" ", normalize-space(@class), " "), " comment-meta ")]')
          meta_info_item = meta_item.search('./div[contains(concat(" ", normalize-space(@class), " "), " comment-info ")]')

          comment_timestamps = parse_timestamps(meta_info_item)
          comment_timestamps["retrieval"] = retrieval_timestamp

          if meta_info_item.at_xpath('./a/@href').nil?
            comment_author = Wagg::Utils::Functions.str_at_xpath(meta_info_item, './strong/text()')
          else
            comment_author = Wagg::Utils::Functions.str_at_xpath(meta_info_item, './a/@href')[/\/user\/(?<author>.+)\/commented/,1]
          end

          # Parse comment's voting meta data
          ballot_item = meta_item.search('./div[contains(concat(" ", normalize-space(@class), " "), " comment-votes-info ")]')<commit_msg>[FIX] Storing comment's body inner HTML instead of Nokogiri nodes (too complicated and prone to errors)<commit_after># encoding: utf-8

require 'wagg/crawler/vote'

module Wagg
  module Crawler
    class Comment
      attr_reader :id, :author, :timestamps, :body
      attr_reader :news_url, :news_index
      attr_accessor :votes_count, :karma

      def initialize(id, author, body, timestamps, news_url, news_index)
        @id = id
        @author = author
        @body = body
        @timestamps = timestamps
        @news_url = news_url
        @news_index = news_index

        @karma = nil
        @votes_count = nil

        @votes = nil

        @votes_closed = (@timestamps['creation'] + Wagg::Utils::Constants::COMMENT_VOTES_LIFETIME) <= @timestamps['retrieval']
      end

      def votes
        if @votes.nil? && self.votes_available?
          # TODO Revise the karma.nil and votes_count.nil case
          #if !@karma.nil? && !@votes_count.nil?
            @votes = Vote.parse_comment_votes(@id)
          #end
        end

        @votes
      end

      def voting_closed?
        @votes_closed
      end

      def voting_open?
        !@votes_closed
      end

      def modified?
        @timestamps.has_key?('edition') && !@timestamps['edition'].nil? && @timestamps['edition'] > 0
      end

      def votes?
        # TODO Revise the karma.nil and votes_count.nil case
        #self.votes_available? && @votes.size > 0
        self.votes_available? && @votes_count > 0
      end

      def votes_available?
        (@timestamps['creation'] <= @timestamps['retrieval']) &&
            (@timestamps['retrieval'] <= (@timestamps['creation'] + Wagg::Utils::Constants::COMMENT_VOTES_LIFETIME + Wagg::Utils::Constants::COMMENT_CONTRIBUTION_LIFETIME))
      end

      def votes_consistent?
        self.votes? ? @votes_count == self.votes.size : FALSE
      end

      def parent_index
        referred_comments_list = @body.search('.//a[contains(concat(" ", normalize-space(@class), " "), " tooltip ")]')
        @body.each { |x| puts x.class }
        puts referred_comments_list
        #referred_comments_list = @body.search('//a[matches(@class, "tooltip c:%{cid}-\d+"]' % {cid: @id})
        parent_comment_item = referred_comments_list.first
        parent_comment_index = Wagg::Utils::Functions.str_at_xpath(@body, './a/text()').to_i

        parent_comment_index
      end

      def position_in_news
        Wagg::Utils::Functions.str_at_xpath(@body, './a/strong/text()')[/(?<position>\d+)/].to_i
      end

      def to_s
        "COMMENT : %{id} - %{a}" % {id:@id, a:@author} +
            "\n" +
            "    %{news_index} - %{news_url}" % {news_index:@news_index, news_url:@news_url} +
            "\n" +
            "    %{ts}" % {ts:@timestamps} +
            "\n" +
            "    %{b}..." % {b:@body[0,20]} +
            "\n" +
            "    %{k} :: %{vc}" % {k:@karma, vc:@votes_count} +
            "\n" +
            "    (%{vc}) => %{v}" % {vc:(@votes.nil? ? 'EMPTY' : @votes.size), v:@votes}
      end

      class << self
        def parse(item, retrieval_timestamp=Time.now.to_i)
          # Parse comment's body data
          body_item = item.search('.//div[contains(concat(" ", normalize-space(@class), " "), " comment-body ")]')
          #comment_body = body_item.search('./child::node()').to_s.scrub.strip
          comment_body = body_item.inner_html.strip
          comment_id = Wagg::Utils::Functions.str_at_xpath(body_item, './@id')[/(?!c-)(?<id>\d+)/].to_i
          # Also available at unique id: //*[@id="cid-XXXXXXXX"]/a/@href
          # TODO: Use regex to remove last element in extraced href instead of these functions...
          comment_news_url = Wagg::Utils::Constants::SITE_URL + Wagg::Utils::Functions.str_at_xpath(body_item, './a/@href').split('/')[0...-1].join('/')
          comment_news_index = Wagg::Utils::Functions.str_at_xpath(item, './@id')[/(?!cid-)(?<id>\d+)/].to_i

          # Parse comment's authorship meta data
          meta_item = item.search('.//div[contains(concat(" ", normalize-space(@class), " "), " comment-meta ")]')
          meta_info_item = meta_item.search('./div[contains(concat(" ", normalize-space(@class), " "), " comment-info ")]')

          comment_timestamps = parse_timestamps(meta_info_item)
          comment_timestamps["retrieval"] = retrieval_timestamp

          if meta_info_item.at_xpath('./a/@href').nil?
            comment_author = Wagg::Utils::Functions.str_at_xpath(meta_info_item, './strong/text()')
          else
            comment_author = Wagg::Utils::Functions.str_at_xpath(meta_info_item, './a/@href')[/\/user\/(?<author>.+)\/commented/,1]
          end

          # Parse comment's voting meta data
          ballot_item = meta_item.search('./div[contains(concat(" ", normalize-space(@class), " "), " comment-votes-info ")]')
{'fraction_non_alphanumeric': 0.16513572092063164, 'fraction_numerical': 0.0020641965115078954, 'mean_word_length': 2.5032537960954446, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3279821', 'n_tokens_mistral': 3295, 'n_tokens_neox': 3203, 'n_words': 546}