<commit_before>
/*!
 * Jade - Lexer
 * Copyright(c) 2010 <NAME> <<EMAIL>>
 * MIT Licensed
 */

/**
 * Initialize `Lexer` with the given `str`.
 *
 * @param {String} str
 * @api private
 */

var Lexer = module.exports = function Lexer(str) {
    this.input = str.replace(/\r\n|\r/g, '\n').replace(/\t/g, '  ');
    this.deferredTokens = [];
    this.lastIndents = 0;
    this.lineno = 1;
};

/**
 * Lexer prototype.
 */

Lexer.prototype = {

    /**
     * Single token lookahead.
     *
     * @return {Object}
     * @api private
     */
    
    get peek(){
        return this.stash = this.advance;
    },
    
    /**
     * Return the next token object.
     *
     * @return {Object}
     * @api private
     */
    
    get advance(){
        var self = this,
        captures;
    
        if (this.stash) {
            var tok = this.stash;
            delete this.stash;
            return tok;
        }
    
        if (this.deferredTokens.length) {
            return this.deferredTokens.shift();
        }
    
        /**
        * Generate token object.
        */
    
        function token(type){
            self.input = self.input.substr(captures[0].length);
            return { 
                type: type,
                line: self.lineno,
                val: captures[1]
            };
        }
    
        // EOS
        if (!this.input.length) {
            if (this.lastIndents-- > 0) {
                return { type: 'outdent', line: this.lineno };
            } else {
                return { type: 'eos', line: this.lineno };
            }
        }
    
        // Comment
        if (captures = /^ *\/\/(-)?([^\n]+)/.exec(this.input)) {<commit_msg>Added Lexer#deferred and Lexer#stashed<commit_after>
/*!
 * Jade - Lexer
 * Copyright(c) 2010 <NAME> <<EMAIL>>
 * MIT Licensed
 */

/**
 * Initialize `Lexer` with the given `str`.
 *
 * @param {String} str
 * @api private
 */

var Lexer = module.exports = function Lexer(str) {
    this.input = str.replace(/\r\n|\r/g, '\n').replace(/\t/g, '  ');
    this.deferredTokens = [];
    this.lastIndents = 0;
    this.lineno = 1;
};

/**
 * Lexer prototype.
 */

Lexer.prototype = {

    /**
     * Single token lookahead.
     *
     * @return {Object}
     * @api private
     */
    
    get peek(){
        return this.stash = this.advance;
    },
    
    /**
     * Get stashed token.
     *
     * @return {Object}
     * @api private
     */
    
    get stashed() {
        if (!this.stash) return;
        var tok = this.stash;
        delete this.stash;
        return tok;
    },
    
    /**
     * Get deferred token.
     *
     * @return {Object}
     * @api private
     */
    
    get deferred() {
        if (!this.deferredTokens.length) return;
        return this.deferredTokens.shift();
    },
    
    /**
     * Return the next token object.
     *
     * @return {Object}
     * @api private
     */
    
    get advance(){
        var self = this,
        captures;
    
        /**
        * Generate token object.
        */
    
        function token(type){
            self.input = self.input.substr(captures[0].length);
            return { 
                type: type,
                line: self.lineno,
                val: captures[1]
            };
        }
    
        // EOS
        if (!this.input.length) {
            if (this.lastIndents-- > 0) {
                return { type: 'outdent', line: this.lineno };
            } else {
                return { type: 'eos', line: this.lineno };
            }
        }
    
        // Comment
        if (captures = /^ *\/\/(-)?([^\n]+)/.exec(this.input)) {
{'fraction_non_alphanumeric': 0.15164047422111937, 'fraction_numerical': 0.004962779156327543, 'mean_word_length': 1.3167305236270754, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '760537', 'n_tokens_mistral': 1198, 'n_tokens_neox': 1151, 'n_words': 245}