<commit_before># -*- coding: utf-8 -*-

require 'kramdown'

class Kramdown::Parser::CfMarkdown < Kramdown::Parser::Kramdown
  @@parsers.delete :block_html
  @@parsers.delete :span_html
  @@parsers.delete :setext_header
  @@parsers.delete :typographic_syms
  @@parsers.delete :smart_quotes

  def initialize(*args)
    super(*args)

    @block_parsers.unshift :email_style_sig
    @span_parsers.unshift :email_style_sig_span

    idx = @block_parsers.index(:setext_header)
    @block_parsers[idx] = :cf_setext_header
    @app_controller = args.last[:app]
  end

  Kernel.silence_warnings do
    CF_SETEXT_HEADER_START = /^(#{OPT_SPACE}[^ \t].*?)#{HEADER_ID}[ \t]*?\n(-|=)+\n/
    FENCED_CODEBLOCK_MATCH = /^((~){3,})\s*?((\S+?)(?:\?\S*)?)?\s*?(?:,\s*?(good|bad)\s*?)?\n(.*?)^\1\2*\s*?\n/m
  end

  define_parser(:cf_setext_header, CF_SETEXT_HEADER_START) unless @@parsers.key?(:cf_setext_header)
  alias parse_cf_setext_header parse_setext_header

  Kernel.silence_warnings do
    SIGNATURE_START = /^-- \n/
  end
  def parse_email_style_sig
    @src.pos += @src.matched_size
    result = @src.scan(/.*/m)

    el = new_block_el(:email_style_sig)

    @tree.children << el
    add_text(result, el)

    true
  end
  define_parser(:email_style_sig, SIGNATURE_START) unless @@parsers.key?(:email_style_sig)

  def parse_email_style_sig_span
    @src.pos += @src.matched_size
    el = new_block_el(:email_style_sig)
    @tree.children << el
    parse_spans(el)

    true
  end
  define_parser(:email_style_sig_span, SIGNATURE_START) unless @@parsers.key?(:email_style_sig_span)

  define_parser(:span_html, /\0/) unless @@parsers.key?(:span_html)
  define_parser(:block_html, /\0/) unless @@parsers.key?(:block_html)
  define_parser(:smart_quotes, /\0/) unless @@parsers.key?(:smart_quotes)
  define_parser(:typographic_syms, /\0/) unless @@parsers.key?(:typographic_syms)

  def parse_html_entity
    start_line_number = @src.current_line_number
    @src.pos += @src.matched_size

    @tree.children << Element.new(:entity, ::Kramdown::Utils::Entities.entity('amp'),
                                  nil, location: start_line_number)
    add_text(@src.matched[1..-1])
  end

  def handle_extension(name, opts, body, type, line_no = nil)
    if name == 'nomarkdown'
      add_text(body) if body.is_a?(String)
      true
    else
      super(name, opts, body, type, line_no)
    end
  end

  def parse_attribute_list(str, opts)
    return if str.strip.empty? || str.strip == ':'
    my_opts = {}

    if str =~ /@([a-zA-Z_-]+)/
      my_opts['lang'] = Regexp.last_match(1)
      str = str.gsub(/@([a-zA-Z_-]+)/, '')
    end

    super(str, opts)

    opts.merge!(my_opts)
  end

  def parse_codeblock_fenced
    if @src.check(self.class::FENCED_CODEBLOCK_MATCH)
      start_line_number = @src.current_line_number
      @src.pos += @src.matched_size
      el = new_block_el(:codeblock, @src[6], nil, location: start_line_number)
      lang = @src[3].to_s.strip
      good_bad = @src[5].to_s.strip

      unless lang.empty?
        el.options[:lang] = lang
        el.attr['class'] = "language-#{@src[4]}"
      end

      unless good_bad.empty?
        el.attr['class'] = (el.attr['class'].to_s + ' ' + good_bad).strip
      end

      @tree.children << el

      true
    else
      false
    end
  end
end

class Kramdown::Converter::CfHtml < Kramdown::Converter::Html
  include LinksHelper

  @@language_replacements = { 'svg' => 'xml', 'html5' => 'html' }

  def initialize(*args)
    super(*args)
    @indent = 0
    @sig_content = nil
    @app = args.last.is_a?(Hash) ? args.last[:app] : nil
    @config_manager = ConfigManager.new if @app.nil?
  end

  def conf(name)
    return @app.conf(name) if @app
    @config_manager.get(name, nil, nil)
  end
<commit_msg>fix: filter out event handler and style attributes<commit_after># -*- coding: utf-8 -*-

require 'kramdown'

class Kramdown::Parser::CfMarkdown < Kramdown::Parser::Kramdown
  @@parsers.delete :block_html
  @@parsers.delete :span_html
  @@parsers.delete :setext_header
  @@parsers.delete :typographic_syms
  @@parsers.delete :smart_quotes

  def initialize(*args)
    super(*args)

    @block_parsers.unshift :email_style_sig
    @span_parsers.unshift :email_style_sig_span

    idx = @block_parsers.index(:setext_header)
    @block_parsers[idx] = :cf_setext_header
    @app_controller = args.last[:app]
  end

  Kernel.silence_warnings do
    CF_SETEXT_HEADER_START = /^(#{OPT_SPACE}[^ \t].*?)#{HEADER_ID}[ \t]*?\n(-|=)+\n/
    FENCED_CODEBLOCK_MATCH = /^((~){3,})\s*?((\S+?)(?:\?\S*)?)?\s*?(?:,\s*?(good|bad)\s*?)?\n(.*?)^\1\2*\s*?\n/m
  end

  define_parser(:cf_setext_header, CF_SETEXT_HEADER_START) unless @@parsers.key?(:cf_setext_header)
  alias parse_cf_setext_header parse_setext_header

  Kernel.silence_warnings do
    SIGNATURE_START = /^-- \n/
  end
  def parse_email_style_sig
    @src.pos += @src.matched_size
    result = @src.scan(/.*/m)

    el = new_block_el(:email_style_sig)

    @tree.children << el
    add_text(result, el)

    true
  end
  define_parser(:email_style_sig, SIGNATURE_START) unless @@parsers.key?(:email_style_sig)

  def parse_email_style_sig_span
    @src.pos += @src.matched_size
    el = new_block_el(:email_style_sig)
    @tree.children << el
    parse_spans(el)

    true
  end
  define_parser(:email_style_sig_span, SIGNATURE_START) unless @@parsers.key?(:email_style_sig_span)

  define_parser(:span_html, /\0/) unless @@parsers.key?(:span_html)
  define_parser(:block_html, /\0/) unless @@parsers.key?(:block_html)
  define_parser(:smart_quotes, /\0/) unless @@parsers.key?(:smart_quotes)
  define_parser(:typographic_syms, /\0/) unless @@parsers.key?(:typographic_syms)

  def parse_html_entity
    start_line_number = @src.current_line_number
    @src.pos += @src.matched_size

    @tree.children << Element.new(:entity, ::Kramdown::Utils::Entities.entity('amp'),
                                  nil, location: start_line_number)
    add_text(@src.matched[1..-1])
  end

  def handle_extension(name, opts, body, type, line_no = nil)
    if name == 'nomarkdown'
      add_text(body) if body.is_a?(String)
      true
    else
      super(name, opts, body, type, line_no)
    end
  end

  def parse_attribute_list(str, opts)
    return if str.strip.empty? || str.strip == ':'
    my_opts = {}

    if str =~ /@([a-zA-Z_-]+)/
      my_opts['lang'] = Regexp.last_match(1)
      str = str.gsub(/@([a-zA-Z_-]+)/, '')
    end

    super(str, opts)

    opts.merge!(my_opts)
  end

  def parse_codeblock_fenced
    if @src.check(self.class::FENCED_CODEBLOCK_MATCH)
      start_line_number = @src.current_line_number
      @src.pos += @src.matched_size
      el = new_block_el(:codeblock, @src[6], nil, location: start_line_number)
      lang = @src[3].to_s.strip
      good_bad = @src[5].to_s.strip

      unless lang.empty?
        el.options[:lang] = lang
        el.attr['class'] = "language-#{@src[4]}"
      end

      unless good_bad.empty?
        el.attr['class'] = (el.attr['class'].to_s + ' ' + good_bad).strip
      end

      @tree.children << el

      true
    else
      false
    end
  end

  def update_attr_with_ial(attr, ial)
    if ial[:refs]
      ial[:refs].each do |ref|
        ref = @alds[ref]
        update_attr_with_ial(attr, ref) if ref
      end
    end

    ial.each do |k, v|
      next if k =~ /^on.*/ || k == 'style'

      if k == IAL_CLASS_ATTR
        attr[k] = (attr[k] || '') << " #{v}"
        attr[k].lstrip!
      elsif k.is_a?(String)
        attr[k] = v
      end
    end
  end
end

class Kramdown::Converter::CfHtml < Kramdown::Converter::Html
  include LinksHelper

  @@language_replacements = { 'svg' => 'xml', 'html5' => 'html' }

  def initialize(*args)
    super(*args)
    @indent = 0
    @sig_content = nil
    @app = args.last.is_a?(Hash) ? args.last[:app] : nil
    @config_manager = ConfigManager.new if @app.nil?
  end

  def conf(name)
    return @app.conf(name) if @app
    @config_manager.get(name, nil, nil)
  end

{'fraction_non_alphanumeric': 0.15641057776663742, 'fraction_numerical': 0.004261185612232109, 'mean_word_length': 3.2742367434386717, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4779568', 'n_tokens_mistral': 3072, 'n_tokens_neox': 2973, 'n_words': 513}