<commit_before># -*- coding: utf-8 -*-

require 'kramdown'

class Kramdown::Parser::CfMarkdown < Kramdown::Parser::Kramdown
  @@parsers.delete :block_html
  @@parsers.delete :span_html
  @@parsers.delete :html_entity
  @@parsers.delete :setext_header

  def initialize(*args)
    super(*args)

    @block_parsers.unshift :email_style_sig
    @span_parsers.unshift :email_style_sig_span

    idx = @block_parsers.index(:setext_header)
    @block_parsers[idx] = :cf_setext_header
  end

  Kernel::silence_warnings {
    CF_SETEXT_HEADER_START = /^(#{OPT_SPACE}[^ \t].*?)#{HEADER_ID}[ \t]*?\n(-|=)+\n/
  }
  define_parser(:cf_setext_header, CF_SETEXT_HEADER_START)  unless @@parsers.has_key?(:cf_setext_header)
  alias_method :parse_cf_setext_header, :parse_setext_header

  Kernel::silence_warnings {
    SIGNATURE_START = /^-- \n/
  }
  def parse_email_style_sig
    @src.pos += @src.matched_size
    result = @src.scan(/.*/m)

    el = new_block_el(:email_style_sig)

    @tree.children << el
    add_text(result, el)

    true
  end
  define_parser(:email_style_sig, SIGNATURE_START) unless @@parsers.has_key?(:email_style_sig)


  def parse_email_style_sig_span
    @src.pos += @src.matched_size
    el = new_block_el(:email_style_sig)
    @tree.children << el
    parse_spans(el)

    true
  end
  define_parser(:email_style_sig_span, SIGNATURE_START) unless @@parsers.has_key?(:email_style_sig_span)

  define_parser(:span_html, /\0/) unless @@parsers.has_key?(:pan_html)
  define_parser(:block_html, /\0/) unless @@parsers.has_key?(:block_html)
  define_parser(:html_entity, /\0/) unless @@parsers.has_key?(:html_entity)
end

class Kramdown::Converter::CfHtml < Kramdown::Converter::Html
  def convert_codeblock(el, indent)
    ret = super(el, indent)
    ret.gsub!(/^<div[^>]*>\n?(.*)<\/div>/m, '\1')

    '<code class="block"><pre>' + ret + '</pre></code>'
  end

  def convert_email_style_sig(el, indent)
    "<span class=\"signature\"><br />\n-- <br />\n" + inner(el, indent) + "</span>"
  end

  def convert_a(el, indent)
    if @options[:no_follow]
      if @options[:root_url].blank? or not el.attr['href'].start_with?(@options[:root_url])
        el.attr['rel'] = 'nofollow'
      end
    end
    super(el, indent)
  end<commit_msg>fix: correctly fix the HTML output of kramdown<commit_after># -*- coding: utf-8 -*-

require 'kramdown'

class Kramdown::Parser::CfMarkdown < Kramdown::Parser::Kramdown
  @@parsers.delete :block_html
  @@parsers.delete :span_html
  @@parsers.delete :html_entity
  @@parsers.delete :setext_header

  def initialize(*args)
    super(*args)

    @block_parsers.unshift :email_style_sig
    @span_parsers.unshift :email_style_sig_span

    idx = @block_parsers.index(:setext_header)
    @block_parsers[idx] = :cf_setext_header
  end

  Kernel::silence_warnings {
    CF_SETEXT_HEADER_START = /^(#{OPT_SPACE}[^ \t].*?)#{HEADER_ID}[ \t]*?\n(-|=)+\n/
  }
  define_parser(:cf_setext_header, CF_SETEXT_HEADER_START)  unless @@parsers.has_key?(:cf_setext_header)
  alias_method :parse_cf_setext_header, :parse_setext_header

  Kernel::silence_warnings {
    SIGNATURE_START = /^-- \n/
  }
  def parse_email_style_sig
    @src.pos += @src.matched_size
    result = @src.scan(/.*/m)

    el = new_block_el(:email_style_sig)

    @tree.children << el
    add_text(result, el)

    true
  end
  define_parser(:email_style_sig, SIGNATURE_START) unless @@parsers.has_key?(:email_style_sig)


  def parse_email_style_sig_span
    @src.pos += @src.matched_size
    el = new_block_el(:email_style_sig)
    @tree.children << el
    parse_spans(el)

    true
  end
  define_parser(:email_style_sig_span, SIGNATURE_START) unless @@parsers.has_key?(:email_style_sig_span)

  define_parser(:span_html, /\0/) unless @@parsers.has_key?(:pan_html)
  define_parser(:block_html, /\0/) unless @@parsers.has_key?(:block_html)
  define_parser(:html_entity, /\0/) unless @@parsers.has_key?(:html_entity)
end

class Kramdown::Converter::CfHtml < Kramdown::Converter::Html
  def convert_codeblock(el, indent)
    ret = super(el, indent)
    ret.gsub!(/^(\s*)<div[^>]*>\n?(.*)<\/div>/m, '\1<code class="block"><pre>\2</pre></code>')
    ret.gsub!(/<pre><code>(.*)<\/code><\/pre>/m, '<code class="block"><pre>\1</pre></code>')

    ret
  end

  def convert_email_style_sig(el, indent)
    "<span class=\"signature\"><br />\n-- <br />\n" + inner(el, indent) + "</span>"
  end

  def convert_a(el, indent)
    if @options[:no_follow]
      if @options[:root_url].blank? or not el.attr['href'].start_with?(@options[:root_url])
        el.attr['rel'] = 'nofollow'
      end
    end
    super(el, indent)
  end
{'fraction_non_alphanumeric': 0.16496856709299804, 'fraction_numerical': 0.00260134402774767, 'mean_word_length': 4.279176201372998, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4779569', 'n_tokens_mistral': 1792, 'n_tokens_neox': 1732, 'n_words': 275}