<commit_before>lexer = require './lexer'
{die} = require './helpers'

# The parser builds an AST from a stream of tokens.
#
# A node in the AST is a JavaScript array whose first item is a string
# indicating the type of node.  The rest of the items are either the children
# or they represent the content of a node.  For instance `(1 + 2) × 3 4` will
# produce the tree:
#
#     ['body',
#       ['expr',
#         ['expr',
#           ['number', '1'],
#           ['symbol', '+'],
#           ['number', '2']],
#         ['symbol', '×'],
#         ['number', '3'],
#         ['number', '4']]]
#
# Note, that at right after parsing stage we don't yet know which symbols
# represent verbs and which represent nouns.  This will be resolved later in
# the `compiler`.

@parse = (aplCode, opts = {}) ->
  tokenStream = lexer.tokenize aplCode

  # A single-token lookahead is used.  Variable `token` stores the upcoming
  # token.
  token = tokenStream.next()

  # `consume(tt)` consumes the upcoming token and returns a truthy value only
  # if its type matches `tt`.  A space-separated value of `tt` matches any of
  # a set of token types.
  consume = (tt) ->
    if token.type in tt.split ' ' then token = tokenStream.next()

  # `demand(tt)` is like `consume(tt)` but intolerant to a mismatch.
  demand = (tt) ->
    if token.type isnt tt
      parserError "Expected token of type '#{tt}' but got '#{token.type}'"
    token = tokenStream.next()
    return

  parserError = (message) ->
    die message,
      name: 'APLParserError'
      file: opts.file
      line: token.startLine
      col: token.startCol
      aplCode: aplCode

  # The parser is a recursive descent parser.  Various `parseXXX()` functions
  # roughly correspond to the set of non-terminals in an imaginary grammar.
  parseBody = ->
    body = ['body']
    loop
      if token.type in ['eof', '}'] then return body
      while consume 'separator newline' then ;
      if token.type in ['eof', '}'] then return body
      expr = parseExpr()
      if consume ':' then expr = ['guard', expr, parseExpr()]
      body.push expr

  parseExpr = ->
    expr = ['expr']
    loop
      item = parseItem()
      if consume '←' then return expr.concat [['assign', item, parseExpr()]]
      expr.push item
      if token.type in ') ] } : ; separator newline eof'.split ' '
        return expr

  parseItem = ->
    item = parseIndexable()
    if consume '['
      item = ['index', item].concat parseIndices()
      demand ']'
    item

  parseIndices = ->
    indices = []
    loop
      if consume ';' then indices.push null
      else if token.type is ']' then (indices.push null; return indices)
      else
        indices.push parseExpr()
        if token.type is ']' then return indices
        demand ';'

  parseIndexable = ->
    t = token
    if consume 'number string symbol embedded' then [t.type, t.value]
    else if consume '(' then (expr = parseExpr(); demand ')'; expr)
    else if consume '{' then (b = parseBody(); demand '}'; ['lambda', b])
    else parserError "Encountered unexpected token of type '#{token.type}'"

  result = parseBody()
  #     'hello'} ⍝ fails 'ParserError'
  demand 'eof'
  result<commit_msg>fixed a test in parser.coffee<commit_after>lexer = require './lexer'
{die} = require './helpers'

# The parser builds an AST from a stream of tokens.
#
# A node in the AST is a JavaScript array whose first item is a string
# indicating the type of node.  The rest of the items are either the children
# or they represent the content of a node.  For instance `(1 + 2) × 3 4` will
# produce the tree:
#
#     ['body',
#       ['expr',
#         ['expr',
#           ['number', '1'],
#           ['symbol', '+'],
#           ['number', '2']],
#         ['symbol', '×'],
#         ['number', '3'],
#         ['number', '4']]]
#
# Note, that at right after parsing stage we don't yet know which symbols
# represent verbs and which represent nouns.  This will be resolved later in
# the `compiler`.

@parse = (aplCode, opts = {}) ->
  tokenStream = lexer.tokenize aplCode

  # A single-token lookahead is used.  Variable `token` stores the upcoming
  # token.
  token = tokenStream.next()

  # `consume(tt)` consumes the upcoming token and returns a truthy value only
  # if its type matches `tt`.  A space-separated value of `tt` matches any of
  # a set of token types.
  consume = (tt) ->
    if token.type in tt.split ' ' then token = tokenStream.next()

  # `demand(tt)` is like `consume(tt)` but intolerant to a mismatch.
  demand = (tt) ->
    if token.type isnt tt
      parserError "Expected token of type '#{tt}' but got '#{token.type}'"
    token = tokenStream.next()
    return

  parserError = (message) ->
    die message,
      name: 'APLParserError'
      file: opts.file
      line: token.startLine
      col: token.startCol
      aplCode: aplCode

  # The parser is a recursive descent parser.  Various `parseXXX()` functions
  # roughly correspond to the set of non-terminals in an imaginary grammar.
  parseBody = ->
    body = ['body']
    loop
      if token.type in ['eof', '}'] then return body
      while consume 'separator newline' then ;
      if token.type in ['eof', '}'] then return body
      expr = parseExpr()
      if consume ':' then expr = ['guard', expr, parseExpr()]
      body.push expr

  parseExpr = ->
    expr = ['expr']
    loop
      item = parseItem()
      if consume '←' then return expr.concat [['assign', item, parseExpr()]]
      expr.push item
      if token.type in ') ] } : ; separator newline eof'.split ' '
        return expr

  parseItem = ->
    item = parseIndexable()
    if consume '['
      item = ['index', item].concat parseIndices()
      demand ']'
    item

  parseIndices = ->
    indices = []
    loop
      if consume ';' then indices.push null
      else if token.type is ']' then (indices.push null; return indices)
      else
        indices.push parseExpr()
        if token.type is ']' then return indices
        demand ';'

  parseIndexable = ->
    t = token
    if consume 'number string symbol embedded' then [t.type, t.value]
    else if consume '(' then (expr = parseExpr(); demand ')'; expr)
    else if consume '{' then (b = parseBody(); demand '}'; ['lambda', b])
    else parserError "Encountered unexpected token of type '#{token.type}'"

  result = parseBody()
  #     'hello'} ⍝ fails
  demand 'eof'
  result
{'fraction_non_alphanumeric': 0.13016917293233082, 'fraction_numerical': 0.002506265664160401, 'mean_word_length': 2.5931344963421497, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '462893', 'n_tokens_mistral': 1969, 'n_tokens_neox': 1907, 'n_words': 736}