<commit_before>require 'rubygems'
require 'builder'

module WikiCloth

class WikiBuffer::HTMLElement < WikiBuffer

  ALLOWED_ELEMENTS = ['a','b','i','img','div','span','sup','sub','strike','s','u','font','big','ref','tt','del',
	'small','blockquote','strong','pre','code','references','ol','li','ul','dd','dt','dl','center',
	'h1','h2','h3','h4','h5','h6','p','table','tr','td','th','tbody','thead','tfoot']
  ALLOWED_ATTRIBUTES = ['src','id','name','style','class','href','start','value','colspan','align','boder',
        'cellpadding','cellspacing']
  ESCAPED_TAGS = [ 'nowiki', 'pre', 'code' ]
  SHORT_TAGS = [ 'meta','br','hr']
  NO_NEED_TO_CLOSE = ['li','p'] + SHORT_TAGS

  def initialize(d="",options={},check=nil)
    super("",options)
    self.buffer_type = "Element"
    @in_quotes = false
    @in_single_quotes = false
    @start_tag = 1
    @tag_check = check unless check.nil?
  end

  def run_globals?
    return ESCAPED_TAGS.include?(self.element_name) ? false : true
  end

  def to_s
    if NO_NEED_TO_CLOSE.include?(self.element_name)
      return "<#{self.element_name} />" if SHORT_TAGS.include?(self.element_name)
      return "</#{self.element_name}><#{self.element_name}>" if @tag_check == self.element_name
    end

    if ESCAPED_TAGS.include?(self.element_name)
      # escape all html inside this element
      self.element_content = self.element_content.gsub('<','&lt;').gsub('>','&gt;')
      # hack to fix <code><nowiki> nested mess
      self.element_content = self.element_content.gsub(/&lt;[\/]*\s*nowiki\s*&gt;/,'')
    end

    lhandler = @options[:link_handler]
    case self.element_name
    when "ref"
      self.element_name = "sup"
      named_ref = self.name_attribute
      ref = lhandler.find_reference_by_name(named_ref) unless named_ref.nil?
      if ref.nil?
        lhandler.references << { :name => named_ref, :value => self.element_content, :count => 0 }
        ref = lhandler.references.last
      end
      ref_id = (named_ref.nil? ? "" : "#{named_ref}_") + "#{lhandler.reference_index(ref)}-#{ref[:count]}"
      self.params << { :name => "id", :value => "cite_ref-#{ref_id}" }
      self.params << { :name => "class", :value => "reference" }
      self.element_content = "[<a href=\"#cite_note-" + (named_ref.nil? ? "" : "#{named_ref}_") + 
	"#{lhandler.reference_index(ref)}\">#{lhandler.reference_index(ref)}</a>]"
      ref[:count] += 1
    when "references"
      ref_count = 0
      self.element_name = "ol"
      self.element_content = lhandler.references.collect { |r| 
        ref_count += 1
        ref_name = (r[:name].nil? ? "" : r[:name].to_slug + "_")
        ret = "<li id=\"cite_note-#{ref_name}#{ref_count}\"><b>"
        1.upto(r[:count]) { |x| ret += "<a href=\"#cite_ref-#{ref_name}#{ref_count}-#{x-1}\">" + 
		(r[:count] == 1 ? "^" : (x-1).to_s(26).tr('0-9a-p', 'a-z')) + "</a> " }
        ret += "</b> #{r[:value]}</li>"
      }.to_s
    when "nowiki"
      return self.element_content
    end

    tmp = elem.tag!(self.element_name, self.element_attributes) { |x| x << self.element_content }
    unless ALLOWED_ELEMENTS.include?(self.element_name)
      tmp.gsub!(/[\-!\|&"\{\}\[\]]/) { |r| self.escape_char(r) }
      return tmp.gsub('<', '&lt;').gsub('>', '&gt;')
    end
    tmp
  end

  def name_attribute
    params.each { |p| return p[:value].to_slug if p.kind_of?(Hash) && p[:name] == "name" }
    return nil
  end

  def element_attributes
    attr = {}
    params.each { |p| attr[p[:name]] = p[:value] if p.kind_of?(Hash) }
    if ALLOWED_ELEMENTS.include?(self.element_name.strip.downcase)
      attr.delete_if { |key,value| !ALLOWED_ATTRIBUTES.include?(key.strip) }
    end
    return attr
  end

  def element_name
    @ename ||= ""
  end

  def element_content
    @econtent ||= ""
  end

  protected

  def escape_char(c)
    c = case c
    when '-' then '&#45;'
    when '!' then '&#33;'
    when '|' then '&#124;'
    when '&' then '&amp;'
    when '"' then '&quot;'
    when '{' then '&#123;'
    when '}' then '&#125;'
    when '[' then '&#91;'
    when ']' then '&#93;'
    when '*' then '&#42;'
    when '#' then '&#35;'
    when ':' then '&#58;'
    when ';' then '&#59;'
    when "'" then '&#39;'
    when '=' then '&#61;'
    else
      c
    end
    return c
  end

  def elem
    Builder::XmlMarkup.new
  end

  def element_name=(val)
    @ename = val
  end

  def element_content=(val)
    @econtent = val
  end

  def in_quotes?
    @in_quotes || @in_single_quotes ? true : false
  end

  def new_char()
    case
    # tag name
    when @start_tag == 1 && current_char == ' '
      self.element_name = self.data.strip.downcase
      self.data = ""
      @start_tag = 2

    # tag is closed <tag/> no attributes
    when @start_tag == 1 && previous_char == '/' && current_char == '>'
      self.data.chop!
      self.element_name = self.data.strip.downcase
      self.data = ""
      @start_tag = 0
      return false
<commit_msg>partial support for includeonly and noinclude tags<commit_after>require 'rubygems'
require 'builder'

module WikiCloth

class WikiBuffer::HTMLElement < WikiBuffer

  ALLOWED_ELEMENTS = ['a','b','i','img','div','span','sup','sub','strike','s','u','font','big','ref','tt','del',
	'small','blockquote','strong','pre','code','references','ol','li','ul','dd','dt','dl','center',
	'h1','h2','h3','h4','h5','h6','p','table','tr','td','th','tbody','thead','tfoot']
  ALLOWED_ATTRIBUTES = ['src','id','name','style','class','href','start','value','colspan','align','boder',
        'cellpadding','cellspacing']
  ESCAPED_TAGS = [ 'nowiki', 'pre', 'code' ]
  SHORT_TAGS = [ 'meta','br','hr']
  NO_NEED_TO_CLOSE = ['li','p'] + SHORT_TAGS

  def initialize(d="",options={},check=nil)
    super("",options)
    self.buffer_type = "Element"
    @in_quotes = false
    @in_single_quotes = false
    @start_tag = 1
    @tag_check = check unless check.nil?
  end

  def run_globals?
    return ESCAPED_TAGS.include?(self.element_name) ? false : true
  end

  def to_s
    if NO_NEED_TO_CLOSE.include?(self.element_name)
      return "<#{self.element_name} />" if SHORT_TAGS.include?(self.element_name)
      return "</#{self.element_name}><#{self.element_name}>" if @tag_check == self.element_name
    end

    if ESCAPED_TAGS.include?(self.element_name)
      # escape all html inside this element
      self.element_content = self.element_content.gsub('<','&lt;').gsub('>','&gt;')
      # hack to fix <code><nowiki> nested mess
      self.element_content = self.element_content.gsub(/&lt;[\/]*\s*nowiki\s*&gt;/,'')
    end

    lhandler = @options[:link_handler]
    case self.element_name
    when "noinclude"
      return self.in_template? ? "" : self.element_content
    when "includeonly"
      return self.in_template? ? self.element_content : ""
    when "ref"
      self.element_name = "sup"
      named_ref = self.name_attribute
      ref = lhandler.find_reference_by_name(named_ref) unless named_ref.nil?
      if ref.nil?
        lhandler.references << { :name => named_ref, :value => self.element_content, :count => 0 }
        ref = lhandler.references.last
      end
      ref_id = (named_ref.nil? ? "" : "#{named_ref}_") + "#{lhandler.reference_index(ref)}-#{ref[:count]}"
      self.params << { :name => "id", :value => "cite_ref-#{ref_id}" }
      self.params << { :name => "class", :value => "reference" }
      self.element_content = "[<a href=\"#cite_note-" + (named_ref.nil? ? "" : "#{named_ref}_") + 
	"#{lhandler.reference_index(ref)}\">#{lhandler.reference_index(ref)}</a>]"
      ref[:count] += 1
    when "references"
      ref_count = 0
      self.element_name = "ol"
      self.element_content = lhandler.references.collect { |r| 
        ref_count += 1
        ref_name = (r[:name].nil? ? "" : r[:name].to_slug + "_")
        ret = "<li id=\"cite_note-#{ref_name}#{ref_count}\"><b>"
        1.upto(r[:count]) { |x| ret += "<a href=\"#cite_ref-#{ref_name}#{ref_count}-#{x-1}\">" + 
		(r[:count] == 1 ? "^" : (x-1).to_s(26).tr('0-9a-p', 'a-z')) + "</a> " }
        ret += "</b> #{r[:value]}</li>"
      }.to_s
    when "nowiki"
      return self.element_content
    end

    tmp = elem.tag!(self.element_name, self.element_attributes) { |x| x << self.element_content }
    unless ALLOWED_ELEMENTS.include?(self.element_name)
      tmp.gsub!(/[\-!\|&"\{\}\[\]]/) { |r| self.escape_char(r) }
      return tmp.gsub('<', '&lt;').gsub('>', '&gt;')
    end
    tmp
  end

  def name_attribute
    params.each { |p| return p[:value].to_slug if p.kind_of?(Hash) && p[:name] == "name" }
    return nil
  end

  def element_attributes
    attr = {}
    params.each { |p| attr[p[:name]] = p[:value] if p.kind_of?(Hash) }
    if ALLOWED_ELEMENTS.include?(self.element_name.strip.downcase)
      attr.delete_if { |key,value| !ALLOWED_ATTRIBUTES.include?(key.strip) }
    end
    return attr
  end

  def element_name
    @ename ||= ""
  end

  def element_content
    @econtent ||= ""
  end

  protected

  def escape_char(c)
    c = case c
    when '-' then '&#45;'
    when '!' then '&#33;'
    when '|' then '&#124;'
    when '&' then '&amp;'
    when '"' then '&quot;'
    when '{' then '&#123;'
    when '}' then '&#125;'
    when '[' then '&#91;'
    when ']' then '&#93;'
    when '*' then '&#42;'
    when '#' then '&#35;'
    when ':' then '&#58;'
    when ';' then '&#59;'
    when "'" then '&#39;'
    when '=' then '&#61;'
    else
      c
    end
    return c
  end

  def elem
    Builder::XmlMarkup.new
  end

  def element_name=(val)
    @ename = val
  end

  def element_content=(val)
    @econtent = val
  end

  def in_template?
    false
  end

  def in_quotes?
    @in_quotes || @in_single_quotes ? true : false
  end

  def new_char()
    case
    # tag name
    when @start_tag == 1 && current_char == ' '
      self.element_name = self.data.strip.downcase
      self.data = ""
      @start_tag = 2

    # tag is closed <tag/> no attributes
    when @start_tag == 1 && previous_char == '/' && current_char == '>'
      self.data.chop!
      self.element_name = self.data.strip.downcase
      self.data = ""
      @start_tag = 0
      return false

{'fraction_non_alphanumeric': 0.1996831996831997, 'fraction_numerical': 0.010296010296010296, 'mean_word_length': 3.1503697617091206, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4320322', 'n_tokens_mistral': 3819, 'n_tokens_neox': 3727, 'n_words': 686}