<commit_before>require "guess_html_encoding/version"

# A small and simple library for guessing the encoding of HTML in Ruby 1.9.
module GuessHtmlEncoding
  # Guess the encoding of an HTML string, using HTTP headers if provided.  HTTP headers can be a string or a hash.
  def self.guess(html, headers = nil)
    html = html.to_s.dup.force_encoding("ASCII-8BIT")
    out = nil

    if headers
      headers = headers.map {|k, v| "#{k}: #{v}" }.join("\n") if headers.is_a?(Hash)
      headers = headers.dup.force_encoding("ASCII-8BIT")
      headers.gsub(/[\r\n]+/, "\n").split("\n").map {|i| i.split(":")}.each do |k,v|
        if k =~ /Content-Type/i && v =~ /charset=([\w\d-]+);?/i
          out = $1.upcase
          break
        end
      end
    end

    if out.nil? || out.empty? || !encoding_loaded?(out)

      out = HTMLScanner.new(html).encoding || out

      out.upcase! unless out.nil?
    end

    # Translate encodings with other names.
    if out
      out = "UTF-8" if %w[DEFAULT UTF8 UNICODE].include?(out)
      out = "CP1251" if out == "CP-1251"
      out = "ISO-8859-1" if %w[LATIN1 LATIN-1].include?(out)
      out = "WINDOWS-1250" if %w[WIN-1251 WIN1251].include?(out)
      out = "GB18030" if %w[GB2312 GB18030].include?(out) 
    end

    out
  end

  # Force an HTML string into a guessed encoding.
  def self.encode(html, headers = nil)
    html_copy = html.to_s.dup
    encoding = guess(html_copy, headers)
    html_copy.force_encoding(encoding_loaded?(encoding) ? encoding : "UTF-8")
    if html_copy.valid_encoding?
      html_copy
    else
      html_copy.force_encoding('ASCII-8BIT').encode('UTF-8', :undef => :replace, :invalid => :replace)
    end
  end

  # Is this encoding loaded?
  def self.encoding_loaded?(encoding)
    !!Encoding.find(encoding) rescue nil
  end

  class HTMLScanner

    def initialize(html)
      @html = html
    end

    # Returns the encoding sniffed from the content of an HTML page, as determined using an 
    # implemention of the algorithm to 'prescan a byte stream to determine its encoding', as
    # specified by the HTML specification: 
    # http://www.w3.org/html/wg/drafts/html/master/syntax.html#prescan-a-byte-stream-to-determine-its-encoding
    def encoding

      position = 0
      charset = nil
      length = @html.length

      done = false

      while position < length && !done

        # First look for a standard HTML comment (ie <!-- blah -->)
        if @html[position, 4] == '<!--'

          position += 2

          position += (@html[position, length].index('-->') || length)

        # Then look for the start of a meta tag
        elsif  @html[position, 6] =~ /\A\<meta[\s\/]/i

          charset, position_increment = charset_from_meta(@html[position + 5, length])

          break if charset

          position += position_increment

        # Then look for <! or </ or <?
        elsif @html[position, 2] =~ /\A\<[\!\/\?]/

          # Advance position to the first > that appears next in string, or end
          position += @html[position, length].index('>') || length

        else
          # Do nothing. (This is just here to make the algorithm easier to follow)
        end

        # Advance position to next character
        position += 1
      end

      charset
    end

    private


    # Given a string which starts with the space or slash following a `<meta`, 
    # look for a charset and returns it along with the position of the next
    # character following the closing `>` character
    def charset_from_meta(string)

      position = 0
      attribute_list = {}
      got_pragma = false
      need_pragma = nil
      charset = nil
      length = string.length

      while position < length

        attribute, position_increment = attribute(string[position, length])
        
        position += position_increment.to_i

        if attribute == nil

          break

        elsif attribute_list[attribute[:attribute_name]]

          # Do nothing
        
        else

          attribute_list[attribute[:attribute_name]] = attribute[:attribute_value]

          if attribute[:attribute_name] == 'http-equiv'
            got_pragma = true

          elsif attribute[:attribute_name] == 'content'
<commit_msg>Minor<commit_after>require "guess_html_encoding/version"

# A small and simple library for guessing the encoding of HTML in Ruby 1.9.
module GuessHtmlEncoding
  # Guess the encoding of an HTML string, using HTTP headers if provided.  HTTP headers can be a string or a hash.
  def self.guess(html, headers = nil)
    html = html.to_s.dup.force_encoding("ASCII-8BIT")
    out = nil

    if headers
      headers = headers.map {|k, v| "#{k}: #{v}" }.join("\n") if headers.is_a?(Hash)
      headers = headers.dup.force_encoding("ASCII-8BIT")
      headers.gsub(/[\r\n]+/, "\n").split("\n").map {|i| i.split(":")}.each do |k,v|
        if k =~ /Content-Type/i && v =~ /charset=([\w\d-]+);?/i
          out = $1.upcase
          break
        end
      end
    end

    if out.nil? || out.empty? || !encoding_loaded?(out)

      out = HTMLScanner.new(html).encoding || out

      out.upcase! unless out.nil?
    end

    # Translate encodings with other names.
    if out
      out = "UTF-8" if %w[DEFAULT UTF8 UNICODE].include?(out)
      out = "CP1251" if out == "CP-1251"
      out = "ISO-8859-1" if %w[LATIN1 LATIN-1].include?(out)
      out = "WINDOWS-1250" if %w[WIN-1251 WIN1251].include?(out)
      out = "GB18030" if %w[GB2312 GB18030].include?(out) 
    end

    out
  end

  # Force an HTML string into a guessed encoding.
  def self.encode(html, headers = nil)
    html_copy = html.to_s.dup
    encoding = guess(html_copy, headers)
    html_copy.force_encoding(encoding_loaded?(encoding) ? encoding : "UTF-8")
    if html_copy.valid_encoding?
      html_copy
    else
      html_copy.force_encoding('ASCII-8BIT').encode('UTF-8', :undef => :replace, :invalid => :replace)
    end
  end

  # Is this encoding loaded?
  def self.encoding_loaded?(encoding)
    !!Encoding.find(encoding) rescue nil
  end

  class HTMLScanner

    def initialize(html)
      @html = html
    end

    # Returns the encoding sniffed from the content of an HTML page, as determined using an 
    # implemention of the algorithm to 'prescan a byte stream to determine its encoding', as
    # specified by the HTML specification: 
    # http://www.w3.org/html/wg/drafts/html/master/syntax.html#prescan-a-byte-stream-to-determine-its-encoding
    def encoding

      position = 0
      charset = nil
      length = @html.length

      done = false

      while position < length && !done

        # First look for a standard HTML comment (ie <!-- blah -->)
        if @html[position, 4] == '<!--'

          position += 2

          position += (@html[position, length].index('-->') || length)

        # Then look for the start of a meta tag
        elsif  @html[position, 6] =~ /\A\<meta[\s\/]/i

          charset, position_increment = charset_from_meta(@html[position + 5, length])

          break if charset

          position += position_increment

        # Then look for <! or </ or <?
        elsif @html[position, 2] =~ /\A\<[\!\/\?]/

          # Advance position to the first > that appears next in string, or end
          position += @html[position, length].index('>') || length

        else
          # Do nothing. (This is just here to make the algorithm easier to follow)
        end

        # Advance position to next character
        position += 1
      end

      charset
    end

    private


    # Given a string which starts with the space or slash following a `<meta`, 
    # look for a charset and returns it along with the position of the next
    # character following the closing `>` character
    def charset_from_meta(string)

      position = 0
      attribute_list = {}
      got_pragma = false
      need_pragma = nil
      charset = nil
      length = string.length

      while position < length

        attribute, position_increment = attribute(string[position, length])
        
        position += position_increment.to_i

        if attribute == nil

          break

        elsif attribute_list[attribute[:attribute_name]]

          # Do nothing
        
        else

          # found a new attribute. Add it to the list
          attribute_list[attribute[:attribute_name]] = attribute[:attribute_value]

          if attribute[:attribute_name] == 'http-equiv'

            got_pragma = true

          elsif attribute[:attribute_name] == 'content'

{'fraction_non_alphanumeric': 0.11514510633298085, 'fraction_numerical': 0.014099400775467043, 'mean_word_length': 2.3472276838379864, 'pattern_counts': {'":': 2, '<': 23, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2236178', 'n_tokens_mistral': 2688, 'n_tokens_neox': 2502, 'n_words': 781}