<commit_before>{Monkey} = require './monkey'

IDENTIFIER = /^[a-zA-Z][a-zA-Z0-9\-]*/

LITERAL = /^[\[\]=\:\-]/

STRING = /^"((?:\\.|[^"])*)"/

MULTI_DENT = /^(?:\n[^\n\S]*)+/

WHITESPACE = /^[^\n\S]+/

class Monkey.Lexer

  tokenize: (code, opts = {}) ->
    @code    = code           # The remainder of the source code.
    @line    = opts.line or 0 # The current line.
    @indent  = 0              # The current indentation level.
    @indebt  = 0              # The over-indentation at the current level.
    @outdebt = 0              # The under-outdentation at the current level.
    @indents = []             # The stack of all current indentation levels.
    @ends    = []             # The stack for pairing up tokens.
    @tokens  = []             # Stream of parsed tokens in the form `['TYPE', value, line]`.

    i = 0
    while @chunk = code.slice i
      i += @identifierToken() or
           #@commentToken()    or
           @whitespaceToken() or
           @lineToken()       or
           @stringToken()     or
           @literalToken()

    while tag = @ends.pop()
      if tag is 'OUTDENT'
        @token 'OUTDENT'
      else
        @error "missing #{tag}"
    @tokens

  whitespaceToken: ->
    if match = WHITESPACE.exec @chunk
      @token 'WHITESPACE', match[0].length
      match[0].length
    else
      0

  token: (tag, value) ->
    @tokens.push [tag, value, @line]

  identifierToken: ->
    if match = IDENTIFIER.exec @chunk
      @token 'IDENTIFIER', match[0]
      match[0].length
    else
      0

  stringToken: ->
    if match = STRING.exec @chunk
      @token 'STRING_LITERAL', match[1]
      match[0].length
    else
      0

  lineToken: ->
    return 0 unless match = MULTI_DENT.exec @chunk


    indent = match[0]
    @line += @count indent, '\n'
    prev = @last @tokens, 1
    size = indent.length - 1 - indent.lastIndexOf '\n'
    diff = size - @indent

    if size is @indent
      @newlineToken()
    else if size > @indent
      @token 'INDENT'
      @indents.push diff
      @ends.push 'OUTDENT'
    else
      while diff < 0
        @error('Should be an OUTDENT, yo') unless @last(@ends) is 'OUTDENT'
        @ends.pop()
        diff += @indents.pop()
        @token 'OUTDENT'
      @token 'TERMINATOR', '\n'
    @indent = size
    indent.length


  literalToken: ->
    if match = LITERAL.exec @chunk
      id = match[0]
      switch id
        when "[" then @token('LPAREN', id)
        when "]" then @token('RPAREN', id)
        when "=" then @token('ASSIGN', id)
        when "-" then @token('INSTRUCT', id)
        when ":" then @token('SCOPE', id)
      1
    else
      @error("WUT??? is '#{@chunk.charAt(0)}'")

  newlineToken: ->
    @token 'TERMINATOR', '\n' unless @tag() is 'TERMINATOR'

  tag: (index, tag) ->
    (tok = @last @tokens, index) and if tag then tok[0] = tag else tok[0]

  value: (index, val) ->
    (tok = @last @tokens, index) and if val then tok[1] = val else tok[1]

  error: (message) ->
    throw SyntaxError "#{message} on line #{ @line + 1}"

  count: (string, substr) ->
    num = pos = 0
    return 1/0 unless substr.length
    num++ while pos = 1 + string.indexOf substr, pos
    num

  last: (array, back) -> array[array.length - (back or 0) - 1]
<commit_msg>Remove unused stuff from lexer<commit_after>{Monkey} = require './monkey'

IDENTIFIER = /^[a-zA-Z][a-zA-Z0-9\-]*/

LITERAL = /^[\[\]=\:\-]/

STRING = /^"((?:\\.|[^"])*)"/

MULTI_DENT = /^(?:\n[^\n\S]*)+/

WHITESPACE = /^[^\n\S]+/

class Monkey.Lexer

  tokenize: (code, opts = {}) ->
    @code    = code           # The remainder of the source code.
    @line    = opts.line or 0 # The current line.
    @indent  = 0              # The current indentation level.
    @indents = []             # The stack of all current indentation levels.
    @ends    = []             # The stack for pairing up tokens.
    @tokens  = []             # Stream of parsed tokens in the form `['TYPE', value, line]`.

    i = 0
    while @chunk = code.slice i
      i += @identifierToken() or
           #@commentToken()    or
           @whitespaceToken() or
           @lineToken()       or
           @stringToken()     or
           @literalToken()

    while tag = @ends.pop()
      if tag is 'OUTDENT'
        @token 'OUTDENT'
      else
        @error "missing #{tag}"
    @tokens

  whitespaceToken: ->
    if match = WHITESPACE.exec @chunk
      @token 'WHITESPACE', match[0].length
      match[0].length
    else
      0

  token: (tag, value) ->
    @tokens.push [tag, value, @line]

  identifierToken: ->
    if match = IDENTIFIER.exec @chunk
      @token 'IDENTIFIER', match[0]
      match[0].length
    else
      0

  stringToken: ->
    if match = STRING.exec @chunk
      @token 'STRING_LITERAL', match[1]
      match[0].length
    else
      0

  lineToken: ->
    return 0 unless match = MULTI_DENT.exec @chunk


    indent = match[0]
    @line += @count indent, '\n'
    prev = @last @tokens, 1
    size = indent.length - 1 - indent.lastIndexOf '\n'
    diff = size - @indent

    if size is @indent
      @newlineToken()
    else if size > @indent
      @token 'INDENT'
      @indents.push diff
      @ends.push 'OUTDENT'
    else
      while diff < 0
        @error('Should be an OUTDENT, yo') unless @last(@ends) is 'OUTDENT'
        @ends.pop()
        diff += @indents.pop()
        @token 'OUTDENT'
      @token 'TERMINATOR', '\n'
    @indent = size
    indent.length


  literalToken: ->
    if match = LITERAL.exec @chunk
      id = match[0]
      switch id
        when "[" then @token('LPAREN', id)
        when "]" then @token('RPAREN', id)
        when "=" then @token('ASSIGN', id)
        when "-" then @token('INSTRUCT', id)
        when ":" then @token('SCOPE', id)
      1
    else
      @error("WUT??? is '#{@chunk.charAt(0)}'")

  newlineToken: ->
    @token 'TERMINATOR', '\n' unless @tag() is 'TERMINATOR'

  tag: (index, tag) ->
    (tok = @last @tokens, index) and if tag then tok[0] = tag else tok[0]

  value: (index, val) ->
    (tok = @last @tokens, index) and if val then tok[1] = val else tok[1]

  error: (message) ->
    throw SyntaxError "#{message} on line #{ @line + 1}"

  count: (string, substr) ->
    num = pos = 0
    return 1/0 unless substr.length
    num++ while pos = 1 + string.indexOf substr, pos
    num

  last: (array, back) -> array[array.length - (back or 0) - 1]

{'fraction_non_alphanumeric': 0.1539670070699136, 'fraction_numerical': 0.010683424980361352, 'mean_word_length': 1.9943555973659455, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3989151', 'n_tokens_mistral': 2237, 'n_tokens_neox': 2142, 'n_words': 632}