<commit_before>module CodeRay module Scanners
  
  # by <NAME>
  class SQL < Scanner

    register_for :sql
    
    RESERVED_WORDS = %w(
      create table index trigger drop primary key set select
      insert update delete replace into
      on from values before and or if exists case when
      then else as group order by avg where
      join inner outer union engine not
      like end using collate show columns begin
    )
    
    PREDEFINED_TYPES = %w(
      char varchar enum binary text tinytext mediumtext
      longtext blob tinyblob mediumblob longblob timestamp
      date time datetime year double decimal float int
      integer tinyint mediumint bigint smallint unsigned bit
      bool boolean hex bin oct
    )
    
    PREDEFINED_FUNCTIONS = %w( sum cast abs pi count min max avg )
    
    DIRECTIVES = %w( auto_increment unique default charset )

    PREDEFINED_CONSTANTS = %w( null true false )
    
    IDENT_KIND = CaseIgnoringWordList.new(:ident).
      add(RESERVED_WORDS, :reserved).
      add(PREDEFINED_TYPES, :pre_type).
      add(PREDEFINED_CONSTANTS, :pre_constant).
      add(PREDEFINED_FUNCTIONS, :predefined).
      add(DIRECTIVES, :directive)
    
    ESCAPE = / [rbfnrtv\n\\\/'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} | . /mx
    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
    
    STRING_PREFIXES = /[xnb]|_\w+/i

    def scan_tokens tokens, options

      state = :initial
      string_type = nil
      string_content = ''

      until eos?

        kind = nil
        match = nil
        
        if state == :initial
          
          if scan(/ \s+ | \\\n /x)
            kind = :space
          
          elsif scan(/^(?:--\s?|#).*/)
            kind = :comment
            
          elsif scan(%r! /\* (?: .*? \*/ | .* ) !mx)
            kind = :comment
            
          elsif scan(/ [-+*\/=<>;,!&^|()\[\]{}~%] | \.(?!\d) /x)
            kind = :operator
            <commit_msg>More cleanup.<commit_after>module CodeRay module Scanners
  
  # by <NAME>
  class SQL < Scanner

    register_for :sql
    
    RESERVED_WORDS = %w(
      create table index trigger drop primary key set select
      insert update delete replace into
      on from values before and or if exists case when
      then else as group order by avg where
      join inner outer union engine not
      like end using collate show columns begin
    )
    
    PREDEFINED_TYPES = %w(
      char varchar enum binary text tinytext mediumtext
      longtext blob tinyblob mediumblob longblob timestamp
      date time datetime year double decimal float int
      integer tinyint mediumint bigint smallint unsigned bit
      bool boolean hex bin oct
    )
    
    PREDEFINED_FUNCTIONS = %w( sum cast abs pi count min max avg )
    
    DIRECTIVES = %w( auto_increment unique default charset )

    PREDEFINED_CONSTANTS = %w( null true false )
    
    IDENT_KIND = CaseIgnoringWordList.new(:ident).
      add(RESERVED_WORDS, :reserved).
      add(PREDEFINED_TYPES, :pre_type).
      add(PREDEFINED_CONSTANTS, :pre_constant).
      add(PREDEFINED_FUNCTIONS, :predefined).
      add(DIRECTIVES, :directive)
    
    ESCAPE = / [rbfnrtv\n\\\/'"] | x[a-fA-F0-9]{1,2} | [0-7]{1,3} | . /mx
    UNICODE_ESCAPE =  / u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8} /x
    
    STRING_PREFIXES = /[xnb]|_\w+/i
    
    def scan_tokens tokens, options
      
      state = :initial
      string_type = nil
      string_content = ''
      
      until eos?
        
        kind = nil
        match = nil
        
        if state == :initial
          
          if scan(/ \s+ | \\\n /x)
            kind = :space
          
          elsif scan(/^(?:--\s?|#).*/)
            kind = :comment
            
          elsif scan(%r! /\* (?: .*? \*/ | .* ) !mx)
            kind = :comment
            
          elsif scan(/ [-+*\/=<>;,!&^|()\[\]{}~%] | \.(?!\d) /x)
            kind = :operator
            
{'fraction_non_alphanumeric': 0.11318344404701074, 'fraction_numerical': 0.007153806847215125, 'mean_word_length': 1.8226387887527036, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4982118', 'n_tokens_mistral': 1316, 'n_tokens_neox': 1295, 'n_words': 355}