<commit_before>'use strict';

var fs = require('fs');
var path = require('path');
var util = require('./util');
var RConsole = require('rich-console');

/**
 * 将文本解析为由常量、插值、代码块3种token组成的列表.
 * @param   {String}cont
 * @parma   {JSON}conf
 * @return  {Array<JSON>}
 */
function parse(content, conf) {
    var data = expandInclude(content, conf);
    var reg  = /(<%=|<%%|<%|%%>|%>|<!--|-->)/;
    var posiData = data.posiData, cont = data.cont;
    var startPosi = 0, endPosi = 0, ctx = [], tokens = [];
    var curDepth = 0, curType, isInCmt = false, mapData = [];

    var getFile = function(posi) {
        return util.getFileDataByPosi(posiData, posi).file;
    };

    var throwParseError = function(tagPosi, errInfo){
        util.throwParseError(errInfo, cont, posiData, startPosi);      
    };

    var throwInvliadTagNestingErr = function(tag2, tag2Posi){
        util.throwInvalidTagNestingError(
            cont, posiData, ctx[0].val, tag2, 
            ctx[0].posi, tag2Posi
        );        
    };

    console.log(cont.length);
    console.log(cont);
    console.log(posiData);
    cont.split(reg).forEach(function(i){
        if(!i || i.length == 0) { return; }

        startPosi = endPosi;
        endPosi  += i.length;

        if(i == '<!--') { 
            if(ctx.length > 0){ throwInvliadTagNestingErr(i, startPosi) }
            ctx.unshift({type: 'comment', val: '<!--', posi: startPosi});
            isInCmt = true;
            return;
        }

        if(i == '<%=') {
            if(ctx.length > 0){ throwInvliadTagNestingErr(i, startPosi) }
            ctx.unshift({type: 'insert', val: '<%=', posi: startPosi});
            return;
        }      

        if(i == '<%') {
            if(ctx.length > 0){ throwInvliadTagNestingErr(i, startPosi) }
            ctx.unshift({type: 'code', val: '<%', posi: startPosi});
            return;
        }     
<commit_msg>注释调试代码<commit_after>'use strict';

var fs = require('fs');
var path = require('path');
var util = require('./util');
var RConsole = require('rich-console');

/**
 * 将文本解析为由常量、插值、代码块3种token组成的列表.
 * @param   {String}cont
 * @parma   {JSON}conf
 * @return  {Array<JSON>}
 */
function parse(content, conf) {
    var data = expandInclude(content, conf);
    var reg  = /(<%=|<%%|<%|%%>|%>|<!--|-->)/;
    var posiData = data.posiData, cont = data.cont;
    var startPosi = 0, endPosi = 0, ctx = [], tokens = [];
    var curDepth = 0, curType, isInCmt = false, mapData = [];

    var getFile = function(posi) {
        return util.getFileDataByPosi(posiData, posi).file;
    };

    var throwParseError = function(tagPosi, errInfo){
        util.throwParseError(errInfo, cont, posiData, startPosi);      
    };

    var throwInvliadTagNestingErr = function(tag2, tag2Posi){
        util.throwInvalidTagNestingError(
            cont, posiData, ctx[0].val, tag2, 
            ctx[0].posi, tag2Posi
        );        
    };

    // console.log(cont.length);
    // console.log(cont);
    // console.log(posiData);
    cont.split(reg).forEach(function(i){
        if(!i || i.length == 0) { return; }

        startPosi = endPosi;
        endPosi  += i.length;

        if(i == '<!--') { 
            if(ctx.length > 0){ throwInvliadTagNestingErr(i, startPosi) }
            ctx.unshift({type: 'comment', val: '<!--', posi: startPosi});
            isInCmt = true;
            return;
        }

        if(i == '<%=') {
            if(ctx.length > 0){ throwInvliadTagNestingErr(i, startPosi) }
            ctx.unshift({type: 'insert', val: '<%=', posi: startPosi});
            return;
        }      

        if(i == '<%') {
            if(ctx.length > 0){ throwInvliadTagNestingErr(i, startPosi) }
            ctx.unshift({type: 'code', val: '<%', posi: startPosi});
            return;
        }     

{'fraction_non_alphanumeric': 0.1749209694415174, 'fraction_numerical': 0.007376185458377239, 'mean_word_length': 2.2564322469982847, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1012697', 'n_tokens_mistral': 1379, 'n_tokens_neox': 1331, 'n_words': 242}