<commit_before># encoding: utf-8

class SanitizeFilter < Filter

  def process(post)
    # Normalize <script> tags so the parser will find them
    post = post.gsub(/<script[\s\/]*/i, '<script ')

    parser = Nokogiri::HTML::DocumentFragment.parse(post)

    remove_unsafe_tags(parser)
    strip_event_handlers(parser)
    enforce_allowscriptaccess(parser)

    parser.to_html
  end

  private

  def remove_unsafe_tags(parser)
    parser.search("script").remove
    parser.search("meta").remove
  end

  def strip_event_handlers(parser)
    parser.search("*").each do |elem|
      elem.attributes.each do |name, attr|
        # XSS fix
        elem.remove_attribute(name) if attr.value && attr.value.downcase.gsub(/[\\]*/, '') =~ /^[\s]*javascript\:/
        # Strip out event handlers
        elem.remove_attribute(name) if name.downcase =~ /^on/
      end
    end
  end

  # Enforces allowScriptAccess = sameDomain on iframes and other embeds.
  def enforce_allowscriptaccess(parser)
    parser.search("*").each do |element|
      change_allowscriptaccess_attribute_on(element)
    end<commit_msg>Remove base tags<commit_after># encoding: utf-8

class SanitizeFilter < Filter

  def process(post)
    # Normalize <script> tags so the parser will find them
    post = post.gsub(/<script[\s\/]*/i, '<script ')

    parser = Nokogiri::HTML::DocumentFragment.parse(post)

    remove_unsafe_tags(parser)
    strip_event_handlers(parser)
    enforce_allowscriptaccess(parser)

    parser.to_html
  end

  private

  def remove_unsafe_tags(parser)
    parser.search("script").remove
    parser.search("meta").remove
    parser.search("base").remove
  end

  def strip_event_handlers(parser)
    parser.search("*").each do |elem|
      elem.attributes.each do |name, attr|
        # XSS fix
        elem.remove_attribute(name) if attr.value && attr.value.downcase.gsub(/[\\]*/, '') =~ /^[\s]*javascript\:/
        # Strip out event handlers
        elem.remove_attribute(name) if name.downcase =~ /^on/
      end
    end
  end

  # Enforces allowScriptAccess = sameDomain on iframes and other embeds.
  def enforce_allowscriptaccess(parser)
    parser.search("*").each do |element|
      change_allowscriptaccess_attribute_on(element)
    end
{'fraction_non_alphanumeric': 0.12153708668453976, 'fraction_numerical': 0.0008936550491510277, 'mean_word_length': 3.3391472868217056, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5912548', 'n_tokens_mistral': 724, 'n_tokens_neox': 697, 'n_words': 152}