<commit_before>/// 
/// Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
/// Use of this file is governed by the BSD 3-clause license that
/// can be found in the LICENSE.txt file in the project root.
/// 


/// 
/// A lexer is recognizer that draws input symbols from a character stream.
/// lexer grammars result in a subclass of this object. A Lexer object
/// uses simplified match() and error recovery mechanisms in the interest
/// of speed.
/// 

import Foundation

open class Lexer: Recognizer<LexerATNSimulator>, TokenSource {
    public static let EOF = -1
    public static let DEFAULT_MODE = 0
    public static let MORE = -2
    public static let SKIP = -3

    public static let DEFAULT_TOKEN_CHANNEL = CommonToken.DEFAULT_CHANNEL
    public static let HIDDEN = CommonToken.HIDDEN_CHANNEL
    public static let MIN_CHAR_VALUE = Character.MIN_VALUE;
    public static let MAX_CHAR_VALUE = Character.MAX_VALUE;

    public var _input: CharStream?
    internal var _tokenFactorySourcePair: TokenSourceAndStream

    /// 
    /// How to create token objects
    /// 
    internal var _factory = CommonTokenFactory.DEFAULT

    /// 
    /// The goal of all lexer rules/methods is to create a token object.
    /// This is an instance variable as multiple rules may collaborate to
    /// create a single token.  nextToken will return this object after
    /// matching lexer rule(s).  If you subclass to allow multiple token
    /// emissions, then set this to the last token to be matched or
    /// something nonnull so that the auto token emit mechanism will not
    /// emit another token.
    /// 
    public var _token: Token?

    /// 
    /// What character index in the stream did the current token start at?
    /// Needed, for example, to get the text for current token.  Set at
    /// the start of nextToken.
    /// 
    public var _tokenStartCharIndex = -1

    /// 
    /// The line on which the first character of the token resides
    /// 
    public var _tokenStartLine = 0

    /// 
    /// The character position of first character within the line
    /// 
    public var _tokenStartCharPositionInLine = 0

    /// 
    /// Once we see EOF on char stream, next token will be EOF.
    /// If you have DONE : EOF ; then you see DONE EOF.
    /// 
    public var _hitEOF = false

    /// 
    /// The channel number for the current token
    /// 
    public var _channel = 0

    /// 
    /// The token type for the current token
    /// 
    public var _type = 0<commit_msg>INVALID_TYPE<commit_after>/// 
/// Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
/// Use of this file is governed by the BSD 3-clause license that
/// can be found in the LICENSE.txt file in the project root.
/// 


/// 
/// A lexer is recognizer that draws input symbols from a character stream.
/// lexer grammars result in a subclass of this object. A Lexer object
/// uses simplified match() and error recovery mechanisms in the interest
/// of speed.
/// 

import Foundation

open class Lexer: Recognizer<LexerATNSimulator>, TokenSource {
    public static let EOF = -1
    public static let DEFAULT_MODE = 0
    public static let MORE = -2
    public static let SKIP = -3

    public static let DEFAULT_TOKEN_CHANNEL = CommonToken.DEFAULT_CHANNEL
    public static let HIDDEN = CommonToken.HIDDEN_CHANNEL
    public static let MIN_CHAR_VALUE = Character.MIN_VALUE;
    public static let MAX_CHAR_VALUE = Character.MAX_VALUE;

    public var _input: CharStream?
    internal var _tokenFactorySourcePair: TokenSourceAndStream

    /// 
    /// How to create token objects
    /// 
    internal var _factory = CommonTokenFactory.DEFAULT

    /// 
    /// The goal of all lexer rules/methods is to create a token object.
    /// This is an instance variable as multiple rules may collaborate to
    /// create a single token.  nextToken will return this object after
    /// matching lexer rule(s).  If you subclass to allow multiple token
    /// emissions, then set this to the last token to be matched or
    /// something nonnull so that the auto token emit mechanism will not
    /// emit another token.
    /// 
    public var _token: Token?

    /// 
    /// What character index in the stream did the current token start at?
    /// Needed, for example, to get the text for current token.  Set at
    /// the start of nextToken.
    /// 
    public var _tokenStartCharIndex = -1

    /// 
    /// The line on which the first character of the token resides
    /// 
    public var _tokenStartLine = 0

    /// 
    /// The character position of first character within the line
    /// 
    public var _tokenStartCharPositionInLine = 0

    /// 
    /// Once we see EOF on char stream, next token will be EOF.
    /// If you have DONE : EOF ; then you see DONE EOF.
    /// 
    public var _hitEOF = false

    /// 
    /// The channel number for the current token
    /// 
    public var _channel = 0

    /// 
    /// The token type for the current token
    /// 
    public var _type = CommonToken.INVALID_TYPE
{'fraction_non_alphanumeric': 0.08010284810126582, 'fraction_numerical': 0.006922468354430379, 'mean_word_length': 2.2625806451612904, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1445818', 'n_tokens_mistral': 1421, 'n_tokens_neox': 1313, 'n_words': 627}