<commit_before>require 'pp'
require 'strscan'

class PuppetLint
  class Lexer
    KEYWORDS = [
      'class',
      'case',
      'default',
      'define',
      'import',
      'if',
      'else',
      'elsif',
      'inherits',
      'node',
      'and',
      'or',
      'undef',
      'true',
      'false',
      'in',
      'unless',
    ]

    KNOWN_TOKENS = [
      [:CLASSREF, /\A(((::){0,1}[A-Z][-\w]*)+)/],
      [:NUMBER, /\A(?:0[xX][0-9A-Fa-f]+|0?\d+(?:\.\d+)?(?:[eE]-?\d+)?)\b/],
      [:NAME, /\A(((::)?[a-z0-9][-\w]*)(::[a-z0-9][-\w]*)*)/],
      [:SSTRING, /\A('.*?')/],
      [:LBRACK, /\A(\[)/],
      [:RBRACK, /\A(\])/],
      [:LBRACE, /\A(\{)/],
      [:RBRACE, /\A(\})/],
      [:LPAREN, /\A(\()/],
      [:RPAREN, /\A(\))/],
      [:ISEQUAL, /\A(==)/],
      [:MATCH, /\A(=~)/],
      [:FARROW, /\A(=>)/],
      [:EQUALS, /\A(=)/],
      [:APPENDS, /\A(\+=)/],
      [:PARROW, /\A(\+>)/],
      [:PLUS, /\A(\+)/],
      [:GREATEREQUAL, /\A(>=)/],
      [:RSHIFT, /\A(>>)/],
      [:GREATERTHAN, /\A(>)/],
      [:LESSEQUAL, /\A(<=)/],
      [:LLCOLLECT, /\A(<<\|)/],
      [:OUT_EDGE, /\A(<-)/],
      [:OUT_EDGE_SUB, /\A(<~)/],
      [:LCOLLECT, /\A(<\|)/],
      [:LSHIFT, /\A(<<)/],
      [:LESSTHAN, /\A(<)/],
      [:NOMATCH, /\A(!~)/],
      [:NOTEQUAL, /\A(!=)/],
      [:NOT, /\A(!)/],
      [:RRCOLLECT, /\A(\|>>)/],
      [:RCOLLECT, /\A(\|>)/],
      [:IN_EDGE, /\A(->)/],
      [:IN_EDGE_SUB, /\A(~>)/],
      [:MINUS, /\A(-)/],
      [:COMMA, /\A(,)/],
      [:DOT, /\A(\.)/],
      [:COLON, /\A(:)/],
      [:AT, /\A(@)/],
      [:SEMIC, /\A(;)/],
      [:QMARK, /\A(\?)/],
      [:BACKSLASH, /\A(\\)/],
      [:DIV, /\A(\/)/],
      [:TIMES, /\A(\*)/],
    ]

    def tokens
      @tokens ||= []
    end

    def tokenise(code)
      code.chomp!

      i = 0

      while i < code.size
        chunk = code[i..-1]

        found = false
        KNOWN_TOKENS.each do |type, regex|
          if value = chunk[regex, 1]
            tokens << new_token(type, value, code[0..i])
            i += value.size
            found = true
            break
          end
        end

        unless found
          if identifier = chunk[/\A([a-z]\w*)/, 1]
            if KEYWORDS.include? identifier
              type = identifier.upcase.to_sym
              tokens << new_token(type, identifier, code[0..i])
            else
              tokens << new_token(:IDENTIFIER, identifier, code[0..i])
            end
            i += identifier.size

          elsif var_name = chunk[/\A\$((::)?([\w-]+::)*[\w-]+)/, 1]
            tokens << new_token(:VARIABLE, var_name, code[0..i])
            i += var_name.size + 1
          
          elsif chunk.match(/\A"/)
            str_contents = StringScanner.new(code[i+1..-1]).scan_until(/[^\\]"/m)
            _ = code[0..i].split("\n")
            interpolate_string(str_contents, _.count, _.last.length)
            i += str_contents.size + 1

          elsif chunk.match(/\A\//)
            str_content = StringScanner.new(code[i+1..-1]).scan_until(/[^\\]\//m)
            tokens << new_token(:REGEX, str_content, code[0..i])
            i += str_content.size + 1

          elsif comment = chunk[/\A(#.*)/, 1]
            comment_size = comment.size
            comment.sub!(/# ?/, '')
            tokens << new_token(:COMMENT, comment, code[0..i])
            i += comment_size

          elsif mlcomment = chunk[/\A(\/\*.*?\*\/)/m, 1]
            mlcomment_size = mlcomment_size
            mlcomment.sub!(/^\/\* ?/, '')
            mlcomment.sub!(/ ?\*\/$/, '')
            tokens << new_token(:MLCOMMENT, mlcomment, code[0..i])
            i += mlcomment_size

          elsif indent = chunk[/\A\n(\s+)/m, 1]<commit_msg>Process keywords early<commit_after>require 'pp'
require 'strscan'

class PuppetLint
  class Lexer
    KEYWORDS = [
      'class',
      'case',
      'default',
      'define',
      'import',
      'if',
      'else',
      'elsif',
      'inherits',
      'node',
      'and',
      'or',
      'undef',
      'true',
      'false',
      'in',
      'unless',
    ]

    KNOWN_TOKENS = [
      [:CLASSREF, /\A(((::){0,1}[A-Z][-\w]*)+)/],
      [:NUMBER, /\A(?:0[xX][0-9A-Fa-f]+|0?\d+(?:\.\d+)?(?:[eE]-?\d+)?)\b/],
      [:NAME, /\A(((::)?[a-z0-9][-\w]*)(::[a-z0-9][-\w]*)*)/],
      [:SSTRING, /\A('.*?')/],
      [:LBRACK, /\A(\[)/],
      [:RBRACK, /\A(\])/],
      [:LBRACE, /\A(\{)/],
      [:RBRACE, /\A(\})/],
      [:LPAREN, /\A(\()/],
      [:RPAREN, /\A(\))/],
      [:ISEQUAL, /\A(==)/],
      [:MATCH, /\A(=~)/],
      [:FARROW, /\A(=>)/],
      [:EQUALS, /\A(=)/],
      [:APPENDS, /\A(\+=)/],
      [:PARROW, /\A(\+>)/],
      [:PLUS, /\A(\+)/],
      [:GREATEREQUAL, /\A(>=)/],
      [:RSHIFT, /\A(>>)/],
      [:GREATERTHAN, /\A(>)/],
      [:LESSEQUAL, /\A(<=)/],
      [:LLCOLLECT, /\A(<<\|)/],
      [:OUT_EDGE, /\A(<-)/],
      [:OUT_EDGE_SUB, /\A(<~)/],
      [:LCOLLECT, /\A(<\|)/],
      [:LSHIFT, /\A(<<)/],
      [:LESSTHAN, /\A(<)/],
      [:NOMATCH, /\A(!~)/],
      [:NOTEQUAL, /\A(!=)/],
      [:NOT, /\A(!)/],
      [:RRCOLLECT, /\A(\|>>)/],
      [:RCOLLECT, /\A(\|>)/],
      [:IN_EDGE, /\A(->)/],
      [:IN_EDGE_SUB, /\A(~>)/],
      [:MINUS, /\A(-)/],
      [:COMMA, /\A(,)/],
      [:DOT, /\A(\.)/],
      [:COLON, /\A(:)/],
      [:AT, /\A(@)/],
      [:SEMIC, /\A(;)/],
      [:QMARK, /\A(\?)/],
      [:BACKSLASH, /\A(\\)/],
      [:DIV, /\A(\/)/],
      [:TIMES, /\A(\*)/],
    ]

    def tokens
      @tokens ||= []
    end

    def tokenise(code)
      code.chomp!

      i = 0

      while i < code.size
        chunk = code[i..-1]

        found = false

        KNOWN_TOKENS.each do |type, regex|
          if value = chunk[regex, 1]
            if type == :NAME
              if KEYWORDS.include? value
                tokens << new_token(value.upcase.to_sym, value, code[0..i])
              else
                tokens << new_token(type, value, code[0..i])
              end
            else
              tokens << new_token(type, value, code[0..i])
            end
            i += value.size
            found = true
            break
          end
        end

        unless found
          if identifier = chunk[/\A([a-z]\w*)/, 1]
            tokens << new_token(:IDENTIFIER, identifier, code[0..i])
            i += identifier.size

          elsif var_name = chunk[/\A\$((::)?([\w-]+::)*[\w-]+)/, 1]
            tokens << new_token(:VARIABLE, var_name, code[0..i])
            i += var_name.size + 1

          elsif chunk.match(/\A"/)
            str_contents = StringScanner.new(code[i+1..-1]).scan_until(/[^\\]"/m)
            _ = code[0..i].split("\n")
            interpolate_string(str_contents, _.count, _.last.length)
            i += str_contents.size + 1

          elsif chunk.match(/\A\//)
            str_content = StringScanner.new(code[i+1..-1]).scan_until(/[^\\]\//m)
            tokens << new_token(:REGEX, str_content, code[0..i])
            i += str_content.size + 1

          elsif comment = chunk[/\A(#.*)/, 1]
            comment_size = comment.size
            comment.sub!(/# ?/, '')
            tokens << new_token(:COMMENT, comment, code[0..i])
            i += comment_size

          elsif mlcomment = chunk[/\A(\/\*.*?\*\/)/m, 1]
            mlcomment_size = mlcomment_size
            mlcomment.sub!(/^\/\* ?/, '')
            mlcomment.sub!(/ ?\*\/$/, '')
            tokens << new_token(:MLCOMMENT, mlcomment, code[0..i])
            i += mlcomment_size

          elsif indent = chunk[/\A\n(\s+)/m, 1]
{'fraction_non_alphanumeric': 0.2712, 'fraction_numerical': 0.008933333333333333, 'mean_word_length': 1.4529103989535643, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7523804', 'n_tokens_mistral': 3088, 'n_tokens_neox': 2821, 'n_words': 482}