<commit_before># The CoffeeScript Lexer. Uses a series of token-matching regexes to attempt
# matches against the beginning of the source code. When a match is found,
# a token is produced, we consume the match, and start again. Tokens are in the
# form:
#
#     [tag, value, locationData]
#
# where locationData is {first_line, first_column, last_line, last_column}, which is a
# format that can be fed directly into [Jison](http://github.com/zaach/jison).  These
# are read by jison in the `parser.lexer` function defined in coffee-script.coffee.

{Rewriter, INVERSES} = require './rewriter'

# Import the helpers we need.
{count, starts, compact, last, repeat, invertLiterate,
locationDataToString,  throwSyntaxError} = require './helpers'

# The Lexer Class
# ---------------

# The Lexer class reads a stream of CoffeeScript and divvies it up into tagged
# tokens. Some potential ambiguity in the grammar has been avoided by
# pushing some extra smarts into the Lexer.
exports.Lexer = class Lexer

  # **tokenize** is the Lexer's main method. Scan by attempting to match tokens
  # one at a time, using a regular expression anchored at the start of the
  # remaining code, or a custom recursive token-matching method
  # (for interpolations). When the next token has been recorded, we move forward
  # within the code past the token, and begin again.
  #
  # Each tokenizing method is responsible for returning the number of characters
  # it has consumed.
  #
  # Before returning the token stream, run it through the [Rewriter](rewriter.html)
  # unless explicitly asked not to.
  tokenize: (code, opts = {}) ->
    @literate   = opts.literate  # Are we lexing literate CoffeeScript?
    @indent     = 0              # The current indentation level.
    @baseIndent = 0              # The overall minimum indentation level
    @indebt     = 0              # The over-indentation at the current level.
    @outdebt    = 0              # The under-outdentation at the current level.
    @indents    = []             # The stack of all current indentation levels.
    @ends       = []             # The stack for pairing up tokens.
    @tokens     = []             # Stream of parsed tokens in the form `['TYPE', value, location data]`.

    @chunkLine =
      opts.line or 0         # The start line for the current @chunk.
    @chunkColumn =
      opts.column or 0       # The start column of the current @chunk.
    code = @clean code         # The stripped, cleaned original source code.

    # At every position, run through this list of attempted matches,
    # short-circuiting if any of them succeed. Their order determines precedence:
    # `@literalToken` is the fallback catch-all.
    i = 0
    while @chunk = code[i..]
      consumed = \
           @identifierToken() or
           @commentToken()    or
           @whitespaceToken() or
           @lineToken()       or
           @heredocToken()    or
           @stringToken()     or
           @numberToken()     or
           @regexToken()      or
           @jsToken()         or
           @literalToken()

      # Update position
      [@chunkLine, @chunkColumn] = @getLineAndColumnFromChunk consumed

      i += consumed

    @closeIndentation()
    @error "missing #{tag}" if tag = @ends.pop()
    return @tokens if opts.rewrite is off
    (new Rewriter).rewrite @tokens

  # Preprocess the code to remove leading and trailing whitespace, carriage
  # returns, etc. If we're lexing literate CoffeeScript, strip external Markdown
  # by removing all lines that aren't indented by at least four spaces or a tab.
  clean: (code) ->
    code = code.slice(1) if code.charCodeAt(0) is BOM
    code = code.replace(/\r/g, '').replace TRAILING_SPACES, ''
    if WHITESPACE.test code
      code = "\n#{code}"
      @chunkLine--
    code = invertLiterate code if @literate
    code

  # Tokenizers
  # ----------

  # Matches identifying literals: variables, keywords, method names, etc.
  # Check to ensure that JavaScript reserved words aren't being used as
  # identifiers. Because CoffeeScript reserves a handful of keywords that are
  # allowed in JavaScript, we're careful not to tag them as keywords when
  # referenced as property names here, so you can still do `jQuery.is()` even
  # though `is` means `===` otherwise.
  identifierToken: ->
    return 0 unless match = IDENTIFIER.exec @chunk
    [input, id, colon] = match

    # Preserve length of id for location data
    idLength = id.length
    poppedToken = undefined

    # This if handles "for own key, value of myObject", loop over keys and values of obj, ignoring proptotype
    if id is 'own' and @tag() is 'FOR'
      @token 'OWN', id
      return id.length

    # colon will be truthy if the identifier ended in a single colon, which is
    # how you define object members in object literal (json) format.
    # In that case, we set "forcedIdentifier" to true. It means, "damn it, this is an identifier, I'm sure of it."
    # But forcedIdentifier can ALSO be true if the previous token was ".", "?.", :: or ?::
    # OR, it can be true if the previos token was NOT SPACED and it is "@". All that makes sense.
    forcedIdentifier = colon or
      (prev = last @tokens) and (prev[0] in ['.', '?.', '::', '?::'] or
      not prev.spaced and prev[0] is '@')
    tag = 'IDENTIFIER'

    # only do this if it is nOT forcedIdentifier. See if the identifier is in JS_KEYWORDS or COFFEE_KEYWORDS
    if not forcedIdentifier and (id in JS_KEYWORDS or id in COFFEE_KEYWORDS)
      tag = id.toUpperCase()
      # if we're using WHEN, and the previous tag was LINE_BREAK, then this is a LEADING_WHEN token.
      if tag is 'WHEN' and @tag() in LINE_BREAK
        tag = 'LEADING_WHEN'
      else if tag is 'FOR' # just make a note we've seen FOR.
        @seenFor = yes
      else if tag is 'UNLESS' # UNLESS is turned into an IF token
        tag = 'IF'
      else if tag in UNARY # meaning, tag is "~" or "!
        tag = 'UNARY'
      else if tag in RELATION # meaning, IN, OF, or INSTANCEOF
        if tag isnt 'INSTANCEOF' and @seenFor
          # tag could be "FORIN" or "FOROF" ()
          # but note, @seenFor is a CLASS variable. So we're REMEMBERING that context across many tokens. So, FOR ... IN and FOR ... OF
          # Yech, i hate the way that's handled.
          tag = 'FOR' + tag
          @seenFor = no
        else
          # here, it could be IN, OF, or INSTANCEOF, and @seenFor was not true.
          tag = 'RELATION' # so all uses of IN, OF and INSTANCEOF outside of FOR are considered RELATIONS.
          # @value() is another function for peeking at the previous token, but it gets the VALUE of that token. So it the value
          # was "!", pop that token off! Change our ID to "!IN", "!FOR" or "!INSTANCEOF" (note, that's our VALUE, not our TAG)
          # so he is merely consolidating the 2 tags into one.
          if @value() is '!'
            poppedToken = @tokens.pop()
            id = '!' + id

    # if id is one of the js or cs reserved words (and a few others), but forcedIndentifier is true, label it an an identifier.
    # otherwise, if it is RESERVED, error out.
    if id in JS_FORBIDDEN
      if forcedIdentifier
        tag = 'IDENTIFIER'
        id  = new String id
        id.reserved = yes
      else if id in RESERVED
        # you can get here if forcedIdentifier is NOT TRUE, and the id is in RESERVED.
        @error "reserved word \"#{id}\""
        # if forcedIdentifier is FALSE, and the id IS in JS_FORBIDDEN, but NOT in RESERVED, you'll keep going. Those possibilities
        # are: JS_KEYWORDS and STRICT_PROSCRIBED, so things like "true, false, typeof" or "eval, arguments"

    # continuing with the case above, if you are using one of the reserved words, and forcedIdentifier is false, go inside this block
    unless forcedIdentifier
      # change id to the coffee alias (resolve the coffee alias) if it is one of those
      id  = COFFEE_ALIAS_MAP[id] if id in COFFEE_ALIASES
      # handle these cases here for some crazy reason.
      tag = switch id
        when '!'                 then 'UNARY'
        when '==', '!='          then 'COMPARE'
        when '&&', '||'          then 'LOGIC'
        when 'true', 'false'     then 'BOOL'
        when 'break', 'continue' then 'STATEMENT'
        else  tag

    # now, we make a token using the tag we have chosen.
    tagToken = @token tag, id, 0, idLength<commit_msg>figured out that == and all that are only in identifierToken due to alias resolution<commit_after># The CoffeeScript Lexer. Uses a series of token-matching regexes to attempt
# matches against the beginning of the source code. When a match is found,
# a token is produced, we consume the match, and start again. Tokens are in the
# form:
#
#     [tag, value, locationData]
#
# where locationData is {first_line, first_column, last_line, last_column}, which is a
# format that can be fed directly into [Jison](http://github.com/zaach/jison).  These
# are read by jison in the `parser.lexer` function defined in coffee-script.coffee.

{Rewriter, INVERSES} = require './rewriter'

# Import the helpers we need.
{count, starts, compact, last, repeat, invertLiterate,
locationDataToString,  throwSyntaxError} = require './helpers'

# The Lexer Class
# ---------------

# The Lexer class reads a stream of CoffeeScript and divvies it up into tagged
# tokens. Some potential ambiguity in the grammar has been avoided by
# pushing some extra smarts into the Lexer.
exports.Lexer = class Lexer

  # **tokenize** is the Lexer's main method. Scan by attempting to match tokens
  # one at a time, using a regular expression anchored at the start of the
  # remaining code, or a custom recursive token-matching method
  # (for interpolations). When the next token has been recorded, we move forward
  # within the code past the token, and begin again.
  #
  # Each tokenizing method is responsible for returning the number of characters
  # it has consumed.
  #
  # Before returning the token stream, run it through the [Rewriter](rewriter.html)
  # unless explicitly asked not to.
  tokenize: (code, opts = {}) ->
    @literate   = opts.literate  # Are we lexing literate CoffeeScript?
    @indent     = 0              # The current indentation level.
    @baseIndent = 0              # The overall minimum indentation level
    @indebt     = 0              # The over-indentation at the current level.
    @outdebt    = 0              # The under-outdentation at the current level.
    @indents    = []             # The stack of all current indentation levels.
    @ends       = []             # The stack for pairing up tokens.
    @tokens     = []             # Stream of parsed tokens in the form `['TYPE', value, location data]`.

    @chunkLine =
      opts.line or 0         # The start line for the current @chunk.
    @chunkColumn =
      opts.column or 0       # The start column of the current @chunk.
    code = @clean code         # The stripped, cleaned original source code.

    # At every position, run through this list of attempted matches,
    # short-circuiting if any of them succeed. Their order determines precedence:
    # `@literalToken` is the fallback catch-all.
    i = 0
    while @chunk = code[i..]
      consumed = \
           @identifierToken() or
           @commentToken()    or
           @whitespaceToken() or
           @lineToken()       or
           @heredocToken()    or
           @stringToken()     or
           @numberToken()     or
           @regexToken()      or
           @jsToken()         or
           @literalToken()

      # Update position
      [@chunkLine, @chunkColumn] = @getLineAndColumnFromChunk consumed

      i += consumed

    @closeIndentation()
    @error "missing #{tag}" if tag = @ends.pop()
    return @tokens if opts.rewrite is off
    (new Rewriter).rewrite @tokens

  # Preprocess the code to remove leading and trailing whitespace, carriage
  # returns, etc. If we're lexing literate CoffeeScript, strip external Markdown
  # by removing all lines that aren't indented by at least four spaces or a tab.
  clean: (code) ->
    code = code.slice(1) if code.charCodeAt(0) is BOM
    code = code.replace(/\r/g, '').replace TRAILING_SPACES, ''
    if WHITESPACE.test code
      code = "\n#{code}"
      @chunkLine--
    code = invertLiterate code if @literate
    code

  # Tokenizers
  # ----------

  # Matches identifying literals: variables, keywords, method names, etc.
  # Check to ensure that JavaScript reserved words aren't being used as
  # identifiers. Because CoffeeScript reserves a handful of keywords that are
  # allowed in JavaScript, we're careful not to tag them as keywords when
  # referenced as property names here, so you can still do `jQuery.is()` even
  # though `is` means `===` otherwise.
  identifierToken: ->
    return 0 unless match = IDENTIFIER.exec @chunk
    [input, id, colon] = match

    # Preserve length of id for location data
    idLength = id.length
    poppedToken = undefined

    # This if handles "for own key, value of myObject", loop over keys and values of obj, ignoring proptotype
    if id is 'own' and @tag() is 'FOR'
      @token 'OWN', id
      return id.length

    # colon will be truthy if the identifier ended in a single colon, which is
    # how you define object members in object literal (json) format.
    # In that case, we set "forcedIdentifier" to true. It means, "damn it, this is an identifier, I'm sure of it."
    # But forcedIdentifier can ALSO be true if the previous token was ".", "?.", :: or ?::
    # OR, it can be true if the previos token was NOT SPACED and it is "@". All that makes sense.
    forcedIdentifier = colon or
      (prev = last @tokens) and (prev[0] in ['.', '?.', '::', '?::'] or
      not prev.spaced and prev[0] is '@')
    tag = 'IDENTIFIER'

    # only do this if it is nOT forcedIdentifier. See if the identifier is in JS_KEYWORDS or COFFEE_KEYWORDS
    if not forcedIdentifier and (id in JS_KEYWORDS or id in COFFEE_KEYWORDS)
      tag = id.toUpperCase()
      # if we're using WHEN, and the previous tag was LINE_BREAK, then this is a LEADING_WHEN token.
      if tag is 'WHEN' and @tag() in LINE_BREAK
        tag = 'LEADING_WHEN'
      else if tag is 'FOR' # just make a note we've seen FOR.
        @seenFor = yes
      else if tag is 'UNLESS' # UNLESS is turned into an IF token
        tag = 'IF'
      else if tag in UNARY # meaning, tag is "~" or "!
        tag = 'UNARY'
      else if tag in RELATION # meaning, IN, OF, or INSTANCEOF
        if tag isnt 'INSTANCEOF' and @seenFor
          # tag could be "FORIN" or "FOROF" ()
          # but note, @seenFor is a CLASS variable. So we're REMEMBERING that context across many tokens. So, FOR ... IN and FOR ... OF
          # Yech, i hate the way that's handled.
          tag = 'FOR' + tag
          @seenFor = no
        else
          # here, it could be IN, OF, or INSTANCEOF, and @seenFor was not true.
          tag = 'RELATION' # so all uses of IN, OF and INSTANCEOF outside of FOR are considered RELATIONS.
          # @value() is another function for peeking at the previous token, but it gets the VALUE of that token. So it the value
          # was "!", pop that token off! Change our ID to "!IN", "!FOR" or "!INSTANCEOF" (note, that's our VALUE, not our TAG)
          # so he is merely consolidating the 2 tags into one.
          if @value() is '!'
            poppedToken = @tokens.pop()
            id = '!' + id

    # if id is one of the js or cs reserved words (and a few others), but forcedIndentifier is true, label it an an identifier.
    # otherwise, if it is RESERVED, error out.
    if id in JS_FORBIDDEN
      if forcedIdentifier
        tag = 'IDENTIFIER'
        id  = new String id
        id.reserved = yes
      else if id in RESERVED
        # you can get here if forcedIdentifier is NOT TRUE, and the id is in RESERVED.
        @error "reserved word \"#{id}\""
        # if forcedIdentifier is FALSE, and the id IS in JS_FORBIDDEN, but NOT in RESERVED, you'll keep going. Those possibilities
        # are: JS_KEYWORDS and STRICT_PROSCRIBED, so things like "true, false, typeof" or "eval, arguments"

    # continuing with the case above, if you are using one of the reserved words, and forcedIdentifier is false, go inside this block
    # but also, of course, if you're not even using a reserved work and forcedIdentifier is false.
    unless forcedIdentifier
      # change id to the coffee alias (resolve the coffee alias) if it is one of those
      id  = COFFEE_ALIAS_MAP[id] if id in COFFEE_ALIASES
      # handle these cases here because they were created by alias resolution!
      tag = switch id
        when '!'                 then 'UNARY'
        when '==', '!='          then 'COMPARE'
        when '&&', '||'          then 'LOGIC'
        when 'true', 'false'     then 'BOOL'
        when 'break', 'continue' then 'STATEMENT'
        else  tag

    # now, we make a token using the tag we have chosen.
    tagToken = @token tag, id, 0, idLength
{'fraction_non_alphanumeric': 0.08661186142102173, 'fraction_numerical': 0.001644157369348209, 'mean_word_length': 2.502159161011721, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4963634', 'n_tokens_mistral': 4935, 'n_tokens_neox': 4777, 'n_words': 2197}