<commit_before>require 'cgi'
require 'bigdecimal'

module Liquid

  module StandardFilters
    HTML_ESCAPE = {
      '&'.freeze => '&amp;'.freeze,
      '>'.freeze => '&gt;'.freeze,
      '<'.freeze => '&lt;'.freeze,
      '"'.freeze => '&quot;'.freeze,
      "'".freeze => '&#39;'.freeze
    }
    HTML_ESCAPE_ONCE_REGEXP = /["><']|&(?!([a-zA-Z]+|(#\d+));)/

    # Return the size of an array or of an string
    def size(input)
      input.respond_to?(:size) ? input.size : 0
    end

    # convert an input string to DOWNCASE
    def downcase(input)
      input.to_s.downcase
    end

    # convert an input string to UPCASE
    def upcase(input)
      input.to_s.upcase
    end

    # capitalize words in the input centence
    def capitalize(input)
      input.to_s.capitalize
    end

    def escape(input)
      CGI.escapeHTML(input) rescue input
    end
    alias_method :h, :escape

    def escape_once(input)
      input.to_s.gsub(HTML_ESCAPE_ONCE_REGEXP, HTML_ESCAPE)
    end

    def url_encode(input)
      CGI.escape(input) rescue input
    end

    def slice(input, offset, length=nil)
      offset = Integer(offset)
      length = length ? Integer(length) : 1
<commit_msg>Auto-untaint variables passed through 'escape'<commit_after>require 'cgi'
require 'bigdecimal'

module Liquid

  module StandardFilters
    HTML_ESCAPE = {
      '&'.freeze => '&amp;'.freeze,
      '>'.freeze => '&gt;'.freeze,
      '<'.freeze => '&lt;'.freeze,
      '"'.freeze => '&quot;'.freeze,
      "'".freeze => '&#39;'.freeze
    }
    HTML_ESCAPE_ONCE_REGEXP = /["><']|&(?!([a-zA-Z]+|(#\d+));)/

    # Return the size of an array or of an string
    def size(input)
      input.respond_to?(:size) ? input.size : 0
    end

    # convert an input string to DOWNCASE
    def downcase(input)
      input.to_s.downcase
    end

    # convert an input string to UPCASE
    def upcase(input)
      input.to_s.upcase
    end

    # capitalize words in the input centence
    def capitalize(input)
      input.to_s.capitalize
    end

    def escape(input)
      CGI.escapeHTML(input).untaint rescue input
    end
    alias_method :h, :escape

    def escape_once(input)
      input.to_s.gsub(HTML_ESCAPE_ONCE_REGEXP, HTML_ESCAPE)
    end

    def url_encode(input)
      CGI.escape(input) rescue input
    end

    def slice(input, offset, length=nil)
      offset = Integer(offset)
      length = length ? Integer(length) : 1

{'fraction_non_alphanumeric': 0.13239553165080678, 'fraction_numerical': 0.0033098882912701694, 'mean_word_length': 2.6143497757847536, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5082975', 'n_tokens_mistral': 884, 'n_tokens_neox': 851, 'n_words': 198}