<commit_before># rubocop:disable Style/ClassVars

require Rails.root + 'lib/cf_kramdown.rb'
require Rails.root + 'lib/cforum_markup.rb'
require Rails.root + 'lib/cf_plaintext.rb'

module ParserHelper
  include CforumMarkup
  include HighlightHelper

  @@parser_modules = {}

  def self.parser_modules
    @@parser_modules
  end

  def md_content
    has_attribute?(:content) ? content.to_s : body.to_s
  end

  def md_format
    has_attribute?(:format) ? format : 'markdown'
  end

  def id_prefix
    'm' + (has_attribute?(:message_id) ? message_id.to_s : priv_message_id.to_s)
  end

  def md_mentions
    nil
  end

  def get_created_at # rubocop:disable Naming/AccessorMethodName
    Time.zone.now
  end

  def highlight_mentions(app, mentions, cnt, do_notify)
    root_path = Rails.application.config.action_controller.relative_url_root || '/'
    already_replaced = {}

    mentions.each do |m|
      next if already_replaced[m[0]]
      already_replaced[m[0]] = true
      username = Regexp.escape(m[0])

      cnt = cnt.gsub(/(\A|[^a-zäöüß0-9_.@\\-])@(#{username})\b/) do
        classes = []
        classes << highlight_notify_mention(m, app) if do_notify

        retval = Regexp.last_match(1) + '[@' + Regexp.last_match(2) +
                 '](' + (root_path + 'users/' + m[1].to_s) + '){: .mention .registered-user'

        if classes.present?
          classes.each do |c|
            next if c.blank?
            retval << classes.join(' ')
          end
        end

        retval + '}'
      end
    end

    cnt
  end

  def to_doc(app, opts = {})
    opts = opts.symbolize_keys!.reverse_merge!(
      input: 'CfMarkdown',
      header_offset: opts[:header_start_index] || app.conf('header_start_index'),
      auto_id_prefix: id_prefix + '-',
      no_follow: true,
      root_url: opts[:root_url] || app.root_url,
      math_engine_opts: { preview: true },
      notify_mentions: true,
      syntax_highlighter: 'rouge',
      with_styles: get_created_at < Time.zone.parse('2017-03-10 15:00')
    )

    if @doc.blank?
      if Rails.env.development?
        load Rails.root + 'lib/cf_kramdown.rb'
        load Rails.root + 'lib/cforum_markup.rb'
      end

      cnt = md_content
      cnt = cforum2markdown(cnt) if md_format == 'cforum'

      mentions = md_mentions
      cnt = highlight_mentions(app, mentions, cnt, opts[:notify_mentions]) if mentions.present?

      cnt.gsub!(/\\@/, '@')

      ncnt = ''
      quote_level = 0

      cnt.lines.each do |l|
        current_ql = 0
        scanner = StringScanner.new(l)
        current_ql += 1 while scanner.scan(/^> ?/)

        if current_ql < quote_level && l !~ /^(?:> ?)*\s*$/m
          ncnt << ('> ' * current_ql) + "\n"
        elsif current_ql > quote_level
          ncnt << ('> ' * quote_level) + "\n"
        end
        quote_level = current_ql

        if l =~ /^(?:> )*~~~\s*(?:\w+)/
          l = l.gsub(/~~~(\s*)(\w+)/) do
            '~~~' + Regexp.last_match(1) + Regexp.last_match(2).downcase
          end
        end

        ncnt << l
      end

      opts[:app] = app
      @doc = Kramdown::Document.new(
        ncnt,
        opts
      )
    end

    @doc
  end

  def to_html(app, opts = {})<commit_msg>ref: sometimes ncnt seems to be frozen?<commit_after># rubocop:disable Style/ClassVars

require Rails.root + 'lib/cf_kramdown.rb'
require Rails.root + 'lib/cforum_markup.rb'
require Rails.root + 'lib/cf_plaintext.rb'

module ParserHelper
  include CforumMarkup
  include HighlightHelper

  @@parser_modules = {}

  def self.parser_modules
    @@parser_modules
  end

  def md_content
    has_attribute?(:content) ? content.to_s : body.to_s
  end

  def md_format
    has_attribute?(:format) ? format : 'markdown'
  end

  def id_prefix
    'm' + (has_attribute?(:message_id) ? message_id.to_s : priv_message_id.to_s)
  end

  def md_mentions
    nil
  end

  def get_created_at # rubocop:disable Naming/AccessorMethodName
    Time.zone.now
  end

  def highlight_mentions(app, mentions, cnt, do_notify)
    root_path = Rails.application.config.action_controller.relative_url_root || '/'
    already_replaced = {}

    mentions.each do |m|
      next if already_replaced[m[0]]
      already_replaced[m[0]] = true
      username = Regexp.escape(m[0])

      cnt = cnt.gsub(/(\A|[^a-zäöüß0-9_.@\\-])@(#{username})\b/) do
        classes = []
        classes << highlight_notify_mention(m, app) if do_notify

        retval = Regexp.last_match(1) + '[@' + Regexp.last_match(2) +
                 '](' + (root_path + 'users/' + m[1].to_s) + '){: .mention .registered-user'

        if classes.present?
          classes.each do |c|
            next if c.blank?
            retval << classes.join(' ')
          end
        end

        retval + '}'
      end
    end

    cnt
  end

  def to_doc(app, opts = {})
    opts = opts.symbolize_keys!.reverse_merge!(
      input: 'CfMarkdown',
      header_offset: opts[:header_start_index] || app.conf('header_start_index'),
      auto_id_prefix: id_prefix + '-',
      no_follow: true,
      root_url: opts[:root_url] || app.root_url,
      math_engine_opts: { preview: true },
      notify_mentions: true,
      syntax_highlighter: 'rouge',
      with_styles: get_created_at < Time.zone.parse('2017-03-10 15:00')
    )

    if @doc.blank?
      if Rails.env.development?
        load Rails.root + 'lib/cf_kramdown.rb'
        load Rails.root + 'lib/cforum_markup.rb'
      end

      cnt = md_content
      cnt = cforum2markdown(cnt) if md_format == 'cforum'

      mentions = md_mentions
      cnt = highlight_mentions(app, mentions, cnt, opts[:notify_mentions]) if mentions.present?

      cnt.gsub!(/\\@/, '@')

      ncnt = ''
      quote_level = 0

      cnt.lines.each do |l|
        current_ql = 0
        scanner = StringScanner.new(l)
        current_ql += 1 while scanner.scan(/^> ?/)

        if current_ql < quote_level && l !~ /^(?:> ?)*\s*$/m
          ncnt << ('> ' * current_ql) + "\n"
        elsif current_ql > quote_level
          ncnt << ('> ' * quote_level) + "\n"
        end
        quote_level = current_ql

        if l =~ /^(?:> )*~~~\s*(?:\w+)/
          l = l.gsub(/~~~(\s*)(\w+)/) do
            '~~~' + Regexp.last_match(1) + Regexp.last_match(2).downcase
          end
        end

        ncnt << l
      end

      ncnt = ncnt.dup if ncnt.frozen?
      opts[:app] = app
      @doc = Kramdown::Document.new(ncnt, opts)
    end

    @doc
  end

  def to_html(app, opts = {})
{'fraction_non_alphanumeric': 0.14099783080260303, 'fraction_numerical': 0.008057018903005888, 'mean_word_length': 2.31025641025641, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4199565', 'n_tokens_mistral': 2387, 'n_tokens_neox': 2259, 'n_words': 441}