<commit_before>
/*!
 * Jade - Language Independent Templates.
 * Copyright(c) 2010 <NAME> <<EMAIL>>
 * MIT Licensed
 */

/**
 * Library version.
 */

exports.version = '0.0.1';

/**
 * Module dependencies.
 */

var fs = require('fs');

/**
 * Initialize jade parser with the given input string.
 *
 * @param {String} str
 * @api public
 */

function Parser(str){
  this.lineno = 0;
  this.input = str;
}

// TODO: handle line endings

Parser.prototype = {
  get nextToken(){
    var self = this,
        captures;
    
    /**
     * Generate token object.
     */
    
    function token(type){
      self.input = self.input.substr(captures[0].length);
      return { 
        type: type,
        line: self.lineno,
        val: captures[1]
      };
    }
    
    // Tag
    if (captures = /^(\w+)/.exec(this.input)) {
      return token('tag');
    }
    
    // Attributes
    if (captures = /^\[([^\]]+)\]/.exec(this.input)) {
      var tok = token('attrs'),
          attrs = tok.val.split(/ *, */);
      tok.attrs = {};
      for (var i = 0, len = attrs.length; i < len; ++i) {
        var pair = attrs[i].split(/ *= */);
        tok.attrs[pair[0]] = pair[1];
      }
      return tok;
    }
    
    // Indent
    if (captures = /^\n( *)/.exec(this.input)) {
      return token('indent');
    }
    
    // Text
    if (captures = /^([^\n]+)/.exec(this.input)) {
      return token('text');
    }
  }
};


var str = fs.readFileSync('examples/layout.jade', 'ascii');

var parser = new Parser(str);
require('sys').puts(require('sys').inspect(parser.nextToken));
require('sys').puts(require('sys').inspect(parser.nextToken));
require('sys').puts(require('sys').inspect(parser.nextToken));
require('sys').puts(require('sys').inspect(parser.nextToken));
require('sys').puts(require('sys').inspect(parser.nextToken));
require('sys').puts(require('sys').inspect(parser.nextToken));
require('sys').puts(require('sys').inspect(parser.nextToken));
require('sys').puts(require('sys').inspect(parser.nextToken));
require('sys').puts(require('sys').inspect(parser.nextToken));<commit_msg>Added EOS<commit_after>
/*!
 * Jade - Language Independent Templates.
 * Copyright(c) 2010 <NAME> <<EMAIL>>
 * MIT Licensed
 */

/**
 * Library version.
 */

exports.version = '0.0.1';

/**
 * Module dependencies.
 */

var fs = require('fs');

/**
 * Initialize jade parser with the given input string.
 *
 * @param {String} str
 * @api public
 */

function Parser(str){
  this.lineno = 0;
  this.input = str;
}

// TODO: handle line endings

Parser.prototype = {
  get nextToken(){
    var self = this,
        captures;
    
    /**
     * Generate token object.
     */
    
    function token(type){
      self.input = self.input.substr(captures[0].length);
      return { 
        type: type,
        line: self.lineno,
        val: captures[1]
      };
    }
    
    // EOS
    if (!this.input.length) {
      return;
    }
    
    // Tag
    if (captures = /^(\w+)/.exec(this.input)) {
      return token('tag');
    }
    
    // Attributes
    if (captures = /^\[([^\]]+)\]/.exec(this.input)) {
      var tok = token('attrs'),
          attrs = tok.val.split(/ *, */);
      tok.attrs = {};
      for (var i = 0, len = attrs.length; i < len; ++i) {
        var pair = attrs[i].split(/ *= */);
        tok.attrs[pair[0]] = pair[1];
      }
      return tok;
    }
    
    // Indent
    if (captures = /^\n( *)/.exec(this.input)) {
      return token('indent');
    }
    
    // Text
    if (captures = /^([^\n]+)/.exec(this.input)) {
      return token('text');
    }
  }
};


var str = fs.readFileSync('examples/layout.jade', 'ascii');

var parser = new Parser(str),
    tok;

while (tok = parser.nextToken) {
  require('sys').puts(require('sys').inspect(tok));
}

{'fraction_non_alphanumeric': 0.20101036958255783, 'fraction_numerical': 0.0069130550385535765, 'mean_word_length': 2.3380656610470276, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7336346', 'n_tokens_mistral': 1323, 'n_tokens_neox': 1292, 'n_words': 252}