<commit_before># encoding: utf-8
require 'strscan'

module CodeRay

  autoload :WordList, 'coderay/helpers/word_list'
  # FIXME: Rename CaseIgnoringWordList to WordList::CaseIgnoring.
  autoload :CaseIgnoringWordList, 'coderay/helpers/word_list'
  
  # = Scanners
  #
  # This module holds the Scanner class and its subclasses.
  # For example, the Ruby scanner is named CodeRay::Scanners::Ruby
  # can be found in coderay/scanners/ruby.
  #
  # Scanner also provides methods and constants for the register
  # mechanism and the [] method that returns the Scanner class
  # belonging to the given lang.
  #
  # See PluginHost.
  module Scanners
    extend PluginHost
    plugin_path File.dirname(__FILE__), 'scanners'
    
    
    # = Scanner
    #
    # The base class for all Scanners.
    #
    # It is a subclass of Ruby's great +StringScanner+, which
    # makes it easy to access the scanning methods inside.
    #
    # It is also +Enumerable+, so you can use it like an Array of
    # Tokens:
    #
    #   require 'coderay'
    #   
    #   c_scanner = CodeRay::Scanners[:c].new "if (*p == '{') nest++;"
    #   
    #   for text, kind in c_scanner
    #     puts text if kind == :operator
    #   end
    #   
    #   # prints: (*==)++;
    #
    # OK, this is a very simple example :)
    # You can also use +map+, +any?+, +find+ and even +sort_by+,
    # if you want.
    class Scanner < StringScanner
      
      extend Plugin
      plugin_host Scanners
      
      # Raised if a Scanner fails while scanning
      ScanError = Class.new Exception
      
      # The default options for all scanner classes.
      #
      # Define @default_options for subclasses.
      DEFAULT_OPTIONS = { }
      
      KINDS_NOT_LOC = [:comment, :doctype, :docstring]
      
      class << self
        
        # Normalizes the given code into a string with UNIX newlines, in the
        # scanner's internal encoding, with invalid and undefined charachters
        # replaced by placeholders. Always returns a new object.
        def normalize code
          original = code
          code = code.to_s unless code.is_a? ::String
          if code.respond_to? :encoding
            code = encode_with_encoding code, self.encoding
          else
            code = to_unix code if code.index ?\r
          end
          # code = code.dup if code.eql? original
          code
        end
        
        # The typical filename suffix for this scanner's language.
        def file_extension extension = plugin_id
          @file_extension ||= extension.to_s
        end
        
        # The encoding used internally by this scanner.
        def encoding name = 'UTF-8'
          @encoding ||= defined?(Encoding.find) && Encoding.find(name)
        end
        
        # The lang of this Scanner class, which is equal to its Plugin ID.
        alias lang plugin_id
        
      protected
        
        def encode_with_encoding code, target_encoding
          if code.encoding == target_encoding
            if code.valid_encoding?
              return to_unix code
            else
              source_encoding = guess_encoding code
            end
          else
            source_encoding = code.encoding
          end
          # print "encode_with_encoding from #{source_encoding} to #{target_encoding}"
          code.encode target_encoding, source_encoding, :universal_newline => true, :undef => :replace, :invalid => :replace
        end
        
        def to_unix code
          code.gsub(/\r\n?/, "\n")
        end
        
        def guess_encoding s
          #:nocov:
          IO.popen("file -b --mime -", "w+") do |file|
            file.write s[0, 1024]
            file.close_write
            begin
              Encoding.find file.gets[/charset=([-\w]+)/, 1]
            rescue ArgumentError
              Encoding::BINARY
            end
          end
          #:nocov:
        end
        <commit_msg>fix warning in Ruby 1.8.6<commit_after># encoding: utf-8
require 'strscan'

module CodeRay

  autoload :WordList, 'coderay/helpers/word_list'
  # FIXME: Rename CaseIgnoringWordList to WordList::CaseIgnoring.
  autoload :CaseIgnoringWordList, 'coderay/helpers/word_list'
  
  # = Scanners
  #
  # This module holds the Scanner class and its subclasses.
  # For example, the Ruby scanner is named CodeRay::Scanners::Ruby
  # can be found in coderay/scanners/ruby.
  #
  # Scanner also provides methods and constants for the register
  # mechanism and the [] method that returns the Scanner class
  # belonging to the given lang.
  #
  # See PluginHost.
  module Scanners
    extend PluginHost
    plugin_path File.dirname(__FILE__), 'scanners'
    
    
    # = Scanner
    #
    # The base class for all Scanners.
    #
    # It is a subclass of Ruby's great +StringScanner+, which
    # makes it easy to access the scanning methods inside.
    #
    # It is also +Enumerable+, so you can use it like an Array of
    # Tokens:
    #
    #   require 'coderay'
    #   
    #   c_scanner = CodeRay::Scanners[:c].new "if (*p == '{') nest++;"
    #   
    #   for text, kind in c_scanner
    #     puts text if kind == :operator
    #   end
    #   
    #   # prints: (*==)++;
    #
    # OK, this is a very simple example :)
    # You can also use +map+, +any?+, +find+ and even +sort_by+,
    # if you want.
    class Scanner < StringScanner
      
      extend Plugin
      plugin_host Scanners
      
      # Raised if a Scanner fails while scanning
      ScanError = Class.new Exception
      
      # The default options for all scanner classes.
      #
      # Define @default_options for subclasses.
      DEFAULT_OPTIONS = { }
      
      KINDS_NOT_LOC = [:comment, :doctype, :docstring]
      
      class << self
        
        # Normalizes the given code into a string with UNIX newlines, in the
        # scanner's internal encoding, with invalid and undefined charachters
        # replaced by placeholders. Always returns a new object.
        def normalize code
          original = code
          code = code.to_s unless code.is_a? ::String
          if code.respond_to? :encoding
            code = encode_with_encoding code, self.encoding
          else
            code = to_unix code if code.index ?\r
          end
          # code = code.dup if code.eql? original
          code
        end
        
        # The typical filename suffix for this scanner's language.
        def file_extension extension = plugin_id
          @file_extension ||= extension.to_s
        end
        
        # The encoding used internally by this scanner.
        def encoding name = 'UTF-8'
          @encoding ||= defined?(Encoding.find) && Encoding.find(name)
        end
        
        # The lang of this Scanner class, which is equal to its Plugin ID.
        alias lang plugin_id
        
      protected
        
        def encode_with_encoding code, target_encoding
          if code.encoding == target_encoding
            if code.valid_encoding?
              return to_unix(code)
            else
              source_encoding = guess_encoding code
            end
          else
            source_encoding = code.encoding
          end
          # print "encode_with_encoding from #{source_encoding} to #{target_encoding}"
          code.encode target_encoding, source_encoding, :universal_newline => true, :undef => :replace, :invalid => :replace
        end
        
        def to_unix code
          code.gsub(/\r\n?/, "\n")
        end
        
        def guess_encoding s
          #:nocov:
          IO.popen("file -b --mime -", "w+") do |file|
            file.write s[0, 1024]
            file.close_write
            begin
              Encoding.find file.gets[/charset=([-\w]+)/, 1]
            rescue ArgumentError
              Encoding::BINARY
            end
          end
          #:nocov:
        end
        
{'fraction_non_alphanumeric': 0.08044664382692551, 'fraction_numerical': 0.0024108615657911434, 'mean_word_length': 1.8393371757925072, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4189671', 'n_tokens_mistral': 2276, 'n_tokens_neox': 2208, 'n_words': 725}