<commit_before>module Tritium
  module Parser
    class Tokenizer
      class Token
        attr_reader :filename, :line_num, :lexeme, :value

        def initialize(filename, line_num, lexeme, value = nil)
          @filename, @line_num = filename, line_num
          @lexeme, @value = lexeme, value
        end

        def to_s
          if @lexeme then
            return @value ? "[#{@lexeme}: #{@value}]" : "[#{lexeme}]"
          else
            return "Error in #{@filename}, line #{@line_num}: #{@value}"
          end
        end

        def ===(symbol)
          @lexeme == symbol
        end
      end

      @@symbols = {
        "=" => :EQUAL, "," => :COMMA,
        "(" => :LPAREN, ")" => :RPAREN,
        "{" => :LBRACE, "}" => :RBRACE
      }
      @@string_matchers = {
        '"' => /^"(\\.|[^"\\])*"/,
        "'" => /^'(\\.|[^'\\])*'/,
        '/' => /^\/(\\.|[^\/\\])*\/[imxouesn]*/
      }

      def initialize(script_string, options = {})
        @filename = options[:filename] || "MAIN"<commit_msg>Fixed a typo.<commit_after>module Tritium
  module Parser
    class Tokenizer
      class Token
        attr_reader :filename, :line_num, :lexeme, :value

        def initialize(filename, line_num, lexeme, value = nil)
          @filename, @line_num = filename, line_num
          @lexeme, @value = lexeme, value
        end

        def to_s
          if @lexeme then
            return @value ? "[#{@lexeme}: #{@value}]" : "[#{@lexeme}]"
          else
            return "Error in #{@filename}, line #{@line_num}: #{@value}"
          end
        end

        def ===(symbol)
          @lexeme == symbol
        end
      end

      @@symbols = {
        "=" => :EQUAL, "," => :COMMA,
        "(" => :LPAREN, ")" => :RPAREN,
        "{" => :LBRACE, "}" => :RBRACE
      }
      @@string_matchers = {
        '"' => /^"(\\.|[^"\\])*"/,
        "'" => /^'(\\.|[^'\\])*'/,
        '/' => /^\/(\\.|[^\/\\])*\/[imxouesn]*/
      }

      def initialize(script_string, options = {})
        @filename = options[:filename] || "MAIN"
{'fraction_non_alphanumeric': 0.2121654501216545, 'fraction_numerical': 0.0, 'mean_word_length': 1.6840731070496084, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '776814', 'n_tokens_mistral': 709, 'n_tokens_neox': 680, 'n_words': 139}