<commit_before>require_dependency 'oneboxer'

class PostAnalyzer

  def initialize(raw, topic_id)
    @raw  = raw
    @topic_id = topic_id
  end

  # What we use to cook posts
  def cook(*args)
    cooked = PrettyText.cook(*args)

    result = Oneboxer.apply(cooked, topic_id: @topic_id) do |url, _|
      Oneboxer.invalidate(url) if args.last[:invalidate_oneboxes]
      Oneboxer.cached_onebox url
    end

    cooked = result.to_html if result.changed?
    cooked
  end

  # How many images are present in the post
  def image_count
    return 0 unless @raw.present?

    cooked_document.search("img").reject do |t|
      dom_class = t["class"]
      if dom_class
        (Post.white_listed_image_classes & dom_class.split(" ")).count > 0
      end
    end.count
  end

  # How many attachments are present in the post
  def attachment_count
    return 0 unless @raw.present?

    attachments = cooked_document.css("a.attachment[href^=\"#{Discourse.store.absolute_base_url}\"]")
    attachments += cooked_document.css("a.attachment[href^=\"#{Discourse.store.relative_base_url}\"]") if Discourse.store.internal?
    attachments.count
  end

  def raw_mentions
    return [] if @raw.blank?
    return @raw_mentions if @raw_mentions.present?

    # strip quotes, code blocks and oneboxes
    cooked_stripped = cooked_document
    cooked_stripped.css("aside.quote").remove
    cooked_stripped.css("pre").remove
    cooked_stripped.css("code").remove
    cooked_stripped.css(".onebox").remove

    raw_mentions = cooked_stripped.css('.mention').map do |e|
       if name = e.inner_text
         name = name[1..-1]
         name.downcase! if name
         name
       end
    end

    raw_mentions.compact!
    raw_mentions.uniq!
    @raw_mention = raw_mentions
  end

  # from rack ... compat with ruby 2.2
  def self.parse_uri_rfc2396(uri)
    @parser ||= defined?(URI::RFC2396_Parser) ? URI::RFC2396_Parser.new : URI
    @parser.parse(uri)
  end

  # Count how many hosts are linked in the post
  def linked_hosts
    return {} if raw_links.blank?
    return @linked_hosts if @linked_hosts.present?

    @linked_hosts = {}
<commit_msg>add missing class<commit_after>require_dependency 'oneboxer'

class PostAnalyzer

  def initialize(raw, topic_id)
    @raw  = raw
    @topic_id = topic_id
  end

  # What we use to cook posts
  def cook(*args)
    cooked = PrettyText.cook(*args)

    result = Oneboxer.apply(cooked, topic_id: @topic_id) do |url, _|
      Oneboxer.invalidate(url) if args.last[:invalidate_oneboxes]
      Oneboxer.cached_onebox url
    end

    cooked = result.to_html if result.changed?
    cooked
  end

  # How many images are present in the post
  def image_count
    return 0 unless @raw.present?

    cooked_document.search("img").reject do |t|
      dom_class = t["class"]
      if dom_class
        (Post.white_listed_image_classes & dom_class.split(" ")).count > 0
      end
    end.count
  end

  # How many attachments are present in the post
  def attachment_count
    return 0 unless @raw.present?

    attachments = cooked_document.css("a.attachment[href^=\"#{Discourse.store.absolute_base_url}\"]")
    attachments += cooked_document.css("a.attachment[href^=\"#{Discourse.store.relative_base_url}\"]") if Discourse.store.internal?
    attachments.count
  end

  def raw_mentions
    return [] if @raw.blank?
    return @raw_mentions if @raw_mentions.present?

    # strip quotes, code blocks and oneboxes
    cooked_stripped = cooked_document
    cooked_stripped.css("aside.quote").remove
    cooked_stripped.css("pre").remove
    cooked_stripped.css("code").remove
    cooked_stripped.css(".onebox").remove

    raw_mentions = cooked_stripped.css('.mention, .mention-group').map do |e|
       if name = e.inner_text
         name = name[1..-1]
         name.downcase! if name
         name
       end
    end

    raw_mentions.compact!
    raw_mentions.uniq!
    @raw_mention = raw_mentions
  end

  # from rack ... compat with ruby 2.2
  def self.parse_uri_rfc2396(uri)
    @parser ||= defined?(URI::RFC2396_Parser) ? URI::RFC2396_Parser.new : URI
    @parser.parse(uri)
  end

  # Count how many hosts are linked in the post
  def linked_hosts
    return {} if raw_links.blank?
    return @linked_hosts if @linked_hosts.present?

    @linked_hosts = {}

{'fraction_non_alphanumeric': 0.10517723880597014, 'fraction_numerical': 0.008861940298507462, 'mean_word_length': 3.13996138996139, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2104186', 'n_tokens_mistral': 1455, 'n_tokens_neox': 1410, 'n_words': 331}