<commit_before># The CoffeeScript language has a good deal of optional syntax, implicit syntax,
# and shorthand syntax. This can greatly complicate a grammar and bloat
# the resulting parse table. Instead of making the parser handle it all, we take
# a series of passes over the token stream, using this **Rewriter** to convert
# shorthand into the unambiguous long form, add implicit indentation and
# parentheses, balance incorrect nestings, and generally clean things up.

# Import the helpers we need.
{include} = require('./helpers').helpers

# The **Rewriter** class is used by the [Lexer](lexer.html), directly against
# its internal array of tokens.
exports.Rewriter = class Rewriter

  # Helpful snippet for debugging:
  #     puts (t[0] + '/' + t[1] for t in @tokens).join ' '

  # Rewrite the token stream in multiple passes, one logical filter at
  # a time. This could certainly be changed into a single pass through the
  # stream, with a big ol' efficient switch, but it's much nicer to work with
  # like this. The order of these passes matters -- indentation must be
  # corrected before implicit parentheses can be wrapped around blocks of code.
  rewrite: (tokens) ->
    @tokens = tokens
    @adjustComments()
    @removeLeadingNewlines()
    @removeMidExpressionNewlines()
    @closeOpenCalls()
    @closeOpenIndexes()
    @addImplicitIndentation()
    @tagPostfixConditionals()
    @addImplicitBraces()
    @addImplicitParentheses()
    @ensureBalance BALANCED_PAIRS
    @rewriteClosingParens()
    @tokens

  # Rewrite the token stream, looking one token ahead and behind.
  # Allow the return value of the block to tell us how many tokens to move
  # forwards (or backwards) in the stream, to make sure we don't miss anything
  # as tokens are inserted and removed, and the stream changes length under
  # our feet.
  scanTokens: (block) ->
    i = 0
    loop
      break unless @tokens[i]
      move = block.call this, @tokens[i], i
      i += move
    true

  detectEnd: (i, condition, action) ->
    levels = 0
    loop
      token = @tokens[i]
      return action.call this, token, i     if levels is 0 and condition.call this, token, i
      return action.call this, token, i - 1 if not token or levels < 0
      levels += 1 if include EXPRESSION_START, token[0]
      levels -= 1 if include EXPRESSION_END, token[0]
      i += 1
    i - 1

  # Massage newlines and indentations so that comments don't have to be
  # correctly indented, or appear on a line of their own.
  adjustComments: ->
    @scanTokens (token, i) ->
      return 1 unless token[0] is 'HERECOMMENT'
      [before, prev, post, after] = [@tokens[i - 2], @tokens[i - 1], @tokens[i + 1], @tokens[i + 2]]
      if after and after[0] is 'INDENT'
        @tokens.splice i + 2, 1
        if before and before[0] is 'OUTDENT' and post and prev[0] is post[0] is 'TERMINATOR'
          @tokens.splice i - 2, 1
        else
          @tokens.splice i, 0, after
      else if prev and prev[0] not in ['TERMINATOR', 'INDENT', 'OUTDENT']
        if post and post[0] is 'TERMINATOR' and after and after[0] is 'OUTDENT'
          @tokens.splice(i + 2, 0, @tokens.splice(i, 2)...)
          if @tokens[i + 2][0] isnt 'TERMINATOR'
            @tokens.splice i + 2, 0, ['TERMINATOR', "\n", prev[2]]
        else
          @tokens.splice i, 0, ['TERMINATOR', "\n", prev[2]]
        return 2
      return 1

  # Leading newlines would introduce an ambiguity in the grammar, so we
  # dispatch them here.
  removeLeadingNewlines: ->
    @tokens.shift() while @tokens[0] and @tokens[0][0] is 'TERMINATOR'

  # Some blocks occur in the middle of expressions -- when we're expecting
  # this, remove their trailing newlines.
  removeMidExpressionNewlines: ->
    @scanTokens (token, i) ->
      return 1 unless include(EXPRESSION_CLOSE, @tag(i + 1)) and token[0] is 'TERMINATOR'
      @tokens.splice i, 1
      return 0

  # The lexer has tagged the opening parenthesis of a method call. Match it with
  # its paired close. We have the mis-nested outdent case included here for
  # calls that close on the same line, just before their outdent.
  closeOpenCalls: ->
    @scanTokens (token, i) ->
      if token[0] is 'CALL_START'
        condition = (token, i) ->
          (token[0] in [')', 'CALL_END']) or (token[0] is 'OUTDENT' and @tokens[i - 1][0] is ')')
        action = (token, i) ->
          idx = if token[0] is 'OUTDENT' then i - 1 else i
          @tokens[idx][0] = 'CALL_END'
        @detectEnd i + 1, condition, action
      return 1

  # The lexer has tagged the opening parenthesis of an indexing operation call.
  # Match it with its paired close.
  closeOpenIndexes: ->
    @scanTokens (token, i) ->
      if token[0] is 'INDEX_START'
        condition = (token, i) -> token[0] in [']', 'INDEX_END']
        action    = (token, i) -> token[0] = 'INDEX_END'
        @detectEnd i + 1, condition, action
      return 1

  # Object literals may be written with implicit braces, for simple cases.
  # Insert the missing braces here, so that the parser doesn't have to.
  addImplicitBraces: ->
    stack = []
    @scanTokens (token, i) ->
      if include EXPRESSION_START, token[0]
        stack.push(if (token[0] is 'INDENT' and (@tag(i - 1) is '{')) then '{' else token[0])
      if include EXPRESSION_END, token[0]
        stack.pop()
      last = stack[stack.length - 1]
      if token[0] is ':' and (not last or last[0] isnt '{')
        stack.push '{'
        idx = if @tag(i - 2) is '@' then i - 2 else i - 1
        tok = ['{', '{', token[2]]
        tok.generated = yes
        @tokens.splice idx, 0, tok
        condition = (token, i) ->
          [one, two, three] = @tokens.slice(i + 1, i + 4)
          return false if 'HERECOMMENT' in [@tag(i + 1), @tag(i - 1)]
          ((token[0] in ['TERMINATOR', 'OUTDENT']) and not ((two and two[0] is ':') or (one and one[0] is '@' and three and three[0] is ':'))) or
            (token[0] is ',' and one and (one[0] not in ['IDENTIFIER', 'STRING', '@', 'TERMINATOR', 'OUTDENT']))
        action = (token, i) ->
          @tokens.splice i, 0, ['}', '}', token[2]]
        @detectEnd i + 2, condition, action
        return 2
      return 1

  # Methods may be optionally called without parentheses, for simple cases.
  # Insert the implicit parentheses here, so that the parser doesn't have to
  # deal with them.
  addImplicitParentheses: ->
    classLine    = no<commit_msg>Fixing issue #700, block comment after first line of class definition.<commit_after># The CoffeeScript language has a good deal of optional syntax, implicit syntax,
# and shorthand syntax. This can greatly complicate a grammar and bloat
# the resulting parse table. Instead of making the parser handle it all, we take
# a series of passes over the token stream, using this **Rewriter** to convert
# shorthand into the unambiguous long form, add implicit indentation and
# parentheses, balance incorrect nestings, and generally clean things up.

# Import the helpers we need.
{include} = require('./helpers').helpers

# The **Rewriter** class is used by the [Lexer](lexer.html), directly against
# its internal array of tokens.
exports.Rewriter = class Rewriter

  # Helpful snippet for debugging:
  #     puts (t[0] + '/' + t[1] for t in @tokens).join ' '

  # Rewrite the token stream in multiple passes, one logical filter at
  # a time. This could certainly be changed into a single pass through the
  # stream, with a big ol' efficient switch, but it's much nicer to work with
  # like this. The order of these passes matters -- indentation must be
  # corrected before implicit parentheses can be wrapped around blocks of code.
  rewrite: (tokens) ->
    @tokens = tokens
    @adjustComments()
    @removeLeadingNewlines()
    @removeMidExpressionNewlines()
    @closeOpenCalls()
    @closeOpenIndexes()
    @addImplicitIndentation()
    @tagPostfixConditionals()
    @addImplicitBraces()
    @addImplicitParentheses()
    @ensureBalance BALANCED_PAIRS
    @rewriteClosingParens()
    @tokens

  # Rewrite the token stream, looking one token ahead and behind.
  # Allow the return value of the block to tell us how many tokens to move
  # forwards (or backwards) in the stream, to make sure we don't miss anything
  # as tokens are inserted and removed, and the stream changes length under
  # our feet.
  scanTokens: (block) ->
    i = 0
    loop
      break unless @tokens[i]
      move = block.call this, @tokens[i], i
      i += move
    true

  detectEnd: (i, condition, action) ->
    levels = 0
    loop
      token = @tokens[i]
      return action.call this, token, i     if levels is 0 and condition.call this, token, i
      return action.call this, token, i - 1 if not token or levels < 0
      levels += 1 if include EXPRESSION_START, token[0]
      levels -= 1 if include EXPRESSION_END, token[0]
      i += 1
    i - 1

  # Massage newlines and indentations so that comments don't have to be
  # correctly indented, or appear on a line of their own.
  adjustComments: ->
    @scanTokens (token, i) ->
      return 1 unless token[0] is 'HERECOMMENT'
      [before, prev, post, after] = [@tokens[i - 2], @tokens[i - 1], @tokens[i + 1], @tokens[i + 2]]
      if after and after[0] is 'INDENT'
        @tokens.splice i + 2, 1
        if before and before[0] is 'OUTDENT' and post and prev[0] is post[0] is 'TERMINATOR'
          @tokens.splice i - 2, 1
        else
          @tokens.splice i, 0, after
      else if prev and prev[0] not in ['TERMINATOR', 'INDENT', 'OUTDENT']
        if post and post[0] is 'TERMINATOR' and after and after[0] is 'OUTDENT'
          @tokens.splice(i + 2, 0, @tokens.splice(i, 2)...)
          if @tokens[i + 2][0] isnt 'TERMINATOR'
            @tokens.splice i + 2, 0, ['TERMINATOR', "\n", prev[2]]
        else
          @tokens.splice i, 0, ['TERMINATOR', "\n", prev[2]]
        return 2
      return 1

  # Leading newlines would introduce an ambiguity in the grammar, so we
  # dispatch them here.
  removeLeadingNewlines: ->
    @tokens.shift() while @tokens[0] and @tokens[0][0] is 'TERMINATOR'

  # Some blocks occur in the middle of expressions -- when we're expecting
  # this, remove their trailing newlines.
  removeMidExpressionNewlines: ->
    @scanTokens (token, i) ->
      return 1 unless include(EXPRESSION_CLOSE, @tag(i + 1)) and token[0] is 'TERMINATOR'
      @tokens.splice i, 1
      return 0

  # The lexer has tagged the opening parenthesis of a method call. Match it with
  # its paired close. We have the mis-nested outdent case included here for
  # calls that close on the same line, just before their outdent.
  closeOpenCalls: ->
    @scanTokens (token, i) ->
      if token[0] is 'CALL_START'
        condition = (token, i) ->
          (token[0] in [')', 'CALL_END']) or (token[0] is 'OUTDENT' and @tokens[i - 1][0] is ')')
        action = (token, i) ->
          idx = if token[0] is 'OUTDENT' then i - 1 else i
          @tokens[idx][0] = 'CALL_END'
        @detectEnd i + 1, condition, action
      return 1

  # The lexer has tagged the opening parenthesis of an indexing operation call.
  # Match it with its paired close.
  closeOpenIndexes: ->
    @scanTokens (token, i) ->
      if token[0] is 'INDEX_START'
        condition = (token, i) -> token[0] in [']', 'INDEX_END']
        action    = (token, i) -> token[0] = 'INDEX_END'
        @detectEnd i + 1, condition, action
      return 1

  # Object literals may be written with implicit braces, for simple cases.
  # Insert the missing braces here, so that the parser doesn't have to.
  addImplicitBraces: ->
    stack = []
    @scanTokens (token, i) ->
      if include EXPRESSION_START, token[0]
        stack.push(if (token[0] is 'INDENT' and (@tag(i - 1) is '{')) then '{' else token[0])
      if include EXPRESSION_END, token[0]
        stack.pop()
      last = stack[stack.length - 1]
      if token[0] is ':' and (not last or last[0] isnt '{')
        stack.push '{'
        idx = if @tag(i - 2) is '@' then i - 2 else i - 1
        idx -= 2 if @tag(idx - 2) is 'HERECOMMENT'
        tok = ['{', '{', token[2]]
        tok.generated = yes
        @tokens.splice idx, 0, tok
        condition = (token, i) ->
          [one, two, three] = @tokens.slice(i + 1, i + 4)
          return false if 'HERECOMMENT' in [@tag(i + 1), @tag(i - 1)]
          ((token[0] in ['TERMINATOR', 'OUTDENT']) and not ((two and two[0] is ':') or (one and one[0] is '@' and three and three[0] is ':'))) or
            (token[0] is ',' and one and (one[0] not in ['IDENTIFIER', 'STRING', '@', 'TERMINATOR', 'OUTDENT']))
        action = (token, i) ->
          @tokens.splice i, 0, ['}', '}', token[2]]
        @detectEnd i + 2, condition, action
        return 2
      return 1

  # Methods may be optionally called without parentheses, for simple cases.
  # Insert the implicit parentheses here, so that the parser doesn't have to
  # deal with them.
  addImplicitParentheses: ->
    classLine    = no
{'fraction_non_alphanumeric': 0.11565203955500618, 'fraction_numerical': 0.014910383189122373, 'mean_word_length': 2.7445762221579404, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '297847', 'n_tokens_mistral': 4193, 'n_tokens_neox': 3973, 'n_words': 1588}