<commit_before># encoding: utf-8
require 'sanitize'

module HTML
  class Pipeline

    # HTML filter with sanization routines and whitelists. This module defines
    # what HTML is allowed in user provided content and fixes up issues with
    # unbalanced tags and whatnot.
    #
    # See the Sanitize docs for more information on the underlying library:
    #
    # https://github.com/rgrove/sanitize/#readme
    #
    # Context options:
    #   :whitelist - The sanitizer whitelist configuration to use. This can be one
    #                of the options constants defined in this class or a custom
    #                sanitize options hash.
    #
    # This filter does not write additional information to the context.
    class SanitizationFilter < Filter

      # The main sanitization whitelist. Only these elements and attributes are
      # allowed through by default.
      WHITELIST = {
        :elements => %w(a abbr b blockquote br cite code dd del dfn div dl dt em
                        h1 h2 h3 h4 h5 h6 hr i img ins kbd li mark meter ol p pre
                        q s samp small source span strong sub sup table tbody td
                        tfooter th thead tr time ul var video wbr),
        :remove_contents => ['script'],
        :attributes => {
          :all         => ['data-after', 'data-id', 'id', 'title', 'class'],
          'a'          => ['href', 'name'],
          'blockquote' => ['cite'],
          'img'        => ['alt', 'height', 'src', 'width'],
          'q'          => ['cite'],
          'source'     => ['src', 'type', 'media'],
          'time'       => ['datetime'],
          'video'      => ['src', 'controls']
        },
        :protocols => {
          'a'          => {'href' => ['ftp', 'http', 'https', 'irc', 'mailto', 'xmpp', 'ed2k', 'magnet', :relative]},
          'blockquote' => {'cite' => ['http', 'https', :relative]},
          'img'        => {'src'  => ['http', 'https', :relative]},
          'q'          => {'cite' => ['http', 'https', :relative]}
        }
      }

      # A more limited sanitization whitelist. This includes all attributes,
      # protocols, and transformers from WHITELIST but with a more locked down
      # set of allowed elements.
      LIMITED = WHITELIST.merge(<commit_msg>linuxfr: add tel: protocol inside whitelist<commit_after># encoding: utf-8
require 'sanitize'

module HTML
  class Pipeline

    # HTML filter with sanization routines and whitelists. This module defines
    # what HTML is allowed in user provided content and fixes up issues with
    # unbalanced tags and whatnot.
    #
    # See the Sanitize docs for more information on the underlying library:
    #
    # https://github.com/rgrove/sanitize/#readme
    #
    # Context options:
    #   :whitelist - The sanitizer whitelist configuration to use. This can be one
    #                of the options constants defined in this class or a custom
    #                sanitize options hash.
    #
    # This filter does not write additional information to the context.
    class SanitizationFilter < Filter

      # The main sanitization whitelist. Only these elements and attributes are
      # allowed through by default.
      WHITELIST = {
        :elements => %w(a abbr b blockquote br cite code dd del dfn div dl dt em
                        h1 h2 h3 h4 h5 h6 hr i img ins kbd li mark meter ol p pre
                        q s samp small source span strong sub sup table tbody td
                        tfooter th thead tr time ul var video wbr),
        :remove_contents => ['script'],
        :attributes => {
          :all         => ['data-after', 'data-id', 'id', 'title', 'class'],
          'a'          => ['href', 'name'],
          'blockquote' => ['cite'],
          'img'        => ['alt', 'height', 'src', 'width'],
          'q'          => ['cite'],
          'source'     => ['src', 'type', 'media'],
          'time'       => ['datetime'],
          'video'      => ['src', 'controls']
        },
        :protocols => {
          'a'          => {'href' => ['ftp', 'http', 'https', 'irc', 'mailto', 'xmpp', 'ed2k', 'magnet', 'tel', :relative]},
          'blockquote' => {'cite' => ['http', 'https', :relative]},
          'img'        => {'src'  => ['http', 'https', :relative]},
          'q'          => {'cite' => ['http', 'https', :relative]}
        }
      }

      # A more limited sanitization whitelist. This includes all attributes,
      # protocols, and transformers from WHITELIST but with a more locked down
      # set of allowed elements.
      LIMITED = WHITELIST.merge(
{'fraction_non_alphanumeric': 0.12524504465258113, 'fraction_numerical': 0.003485079503376171, 'mean_word_length': 1.9798831927319922, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 43, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6802332', 'n_tokens_mistral': 1295, 'n_tokens_neox': 1253, 'n_words': 483}