<commit_before>#!/usr/bin/env ruby

require 'htmlelement'
require 'pseudohiki/inlineparser'
require 'pseudohiki/htmlformat'
#require('hikiparser/hikiblockparser')

module PseudoHiki
  class HtmlFormat
    class << Formatter[PluginNode]
      def visit(leaf)
        escape_inline_tags(leaf) { HtmlPlugin.new(@element_name,leaf.join).apply }
      end
    end
  end
  
  class HtmlPlugin

    PLUGIN_PAT = /^(\w+)([\s\(]+)/
    NUMBER_RE = /(\d+)/

    def parse(data)
      result = nil
      if PLUGIN_PAT =~ data
        @plugin_name = $1
        @with_paren = true if $2.chomp == "("
        result = data.chomp.sub(PLUGIN_PAT,"")
        result[-1,1] = "" if @with_paren
      else
        @plugin_name = data.chomp
        result = ""
      end
      result
    end

    def initialize(tag_type,parsed_data)<commit_msg>an ugly bug fix of htmlplugin.rb<commit_after>#!/usr/bin/env ruby

require 'htmlelement'
require 'pseudohiki/inlineparser'
require 'pseudohiki/htmlformat'
#require('hikiparser/hikiblockparser')

module PseudoHiki
  class HtmlFormat
    class << Formatter[PluginNode]
      def visit(leaf)
        escape_inline_tags(leaf) { HtmlPlugin.new(@element_name,leaf.join).apply }
      end
    end
  end
  
  class XhtmlFormat < HtmlFormat
    Formatter = HtmlFormat::Formatter.dup
    setup_new_formatter(Formatter, XhtmlElement)
  end

  class Xhtml5Format < XhtmlFormat
    Formatter = HtmlFormat::Formatter.dup
    setup_new_formatter(Formatter, Xhtml5Element)
  end

  class HtmlPlugin

    PLUGIN_PAT = /^(\w+)([\s\(]+)/
    NUMBER_RE = /(\d+)/

    def parse(data)
      result = nil
      if PLUGIN_PAT =~ data
        @plugin_name = $1
        @with_paren = true if $2.chomp == "("
        result = data.chomp.sub(PLUGIN_PAT,"")
        result[-1,1] = "" if @with_paren
      else
        @plugin_name = data.chomp
        result = ""
      end
      result
    end

    def initialize(tag_type,parsed_data)
{'fraction_non_alphanumeric': 0.11645962732919254, 'fraction_numerical': 0.005175983436853002, 'mean_word_length': 2.7101727447216892, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '333108', 'n_tokens_mistral': 697, 'n_tokens_neox': 661, 'n_words': 130}