<commit_before>var fs = require("fs.extra"),
    async = require("async"),
    marked = require("marked"),
    path = require('path'),
    gitdata = require(global.rootPath + '/utils/gitdata');

var renderer = new marked.Renderer(), highlightjs = require(global.rootPath + '/utils/highlight');
renderer.code = function (code, language) {
    code = code.replace(/<font class="kw">(((?!<\/font>).)*)<\/font>/g, function(s, s1){  
        return '__$$$'+s1+'$$$__';
    });
    // Check whether the given language is valid for highlight.js.
    var validLang = !!(language && highlightjs.getLanguage(language));
    // Highlight only if the language is valid.
    var highlighted = validLang ? highlightjs.highlight(language, code).value : code;
    highlighted = highlighted.replace(/__\$\$\$(((?!\$\$\$__).)*)\$\$\$__/g, function(s, s1){
        return '<font class="kw">'+s1+'</font>';
    });
    // Render the highlighted code with `hljs` class.
    return '<pre><code class="hljs ' + language + '">' + highlighted + '</code></pre>';
};
var tableCss = 'table table-striped';
renderer.table = function (header, body) {
    return '<table class="' + tableCss + '">' + header + body + '</table>';
};

/*renderer.heading = function (text, level) {
    return '<h'+level+' data-category="">'+text+'</h'+level+'>';
};*/

var markedFilter = function (str) {
    str = marked(str.replace(/(<h[^>]*id\=[^>]*)>/g, function (s, s1) {
        if (s.indexOf('data-category') > -1) return s;
        return s1 + ' data-category=""' + '>';
    })).replace(/\<table\>/g, '<table class="' + tableCss + '">');
    return str;
};

var keywordHighlight = function(k, file){
    if(k==='') return file;
    var kws = k.split(' ');
    for(var i=0,l=kws.length;i<l;i++){
        var reg = new RegExp(kws[i], 'g');
        file = file.replace(reg, function(s){
            return '<font class="kw">'+s+'</font>';
        });
    }
    return file;
};

marked.setOptions({
    renderer: renderer
});

var formateRender = function(obj){
    obj.config = global.config;
    return obj;
};

exports.index = function (req, res) {
    var k = (req.query.k || '').trim();
    var kws = k.replace(/[ ]+/g, ' ').split(' ');
    gitdata.getList(function (list) {
        if (k === '') {
            res.render('index.html', formateRender({ list: list, k: k }));
            return;
        }
        var arr = [];
        for (var i = 0, l = (list || []).length; i < l; i++) {<commit_msg>修复搜索关键字包含正则关键字的bug<commit_after>var fs = require("fs.extra"),
    async = require("async"),
    marked = require("marked"),
    path = require('path'),
    gitdata = require(global.rootPath + '/utils/gitdata');

var renderer = new marked.Renderer(), highlightjs = require(global.rootPath + '/utils/highlight');
renderer.code = function (code, language) {
    code = code.replace(/<font class="kw">(((?!<\/font>).)*)<\/font>/g, function(s, s1){  
        return '__$$$'+s1+'$$$__';
    });
    // Check whether the given language is valid for highlight.js.
    var validLang = !!(language && highlightjs.getLanguage(language));
    // Highlight only if the language is valid.
    var highlighted = validLang ? highlightjs.highlight(language, code).value : code;
    highlighted = highlighted.replace(/__\$\$\$(((?!\$\$\$__).)*)\$\$\$__/g, function(s, s1){
        return '<font class="kw">'+s1+'</font>';
    });
    // Render the highlighted code with `hljs` class.
    return '<pre><code class="hljs ' + language + '">' + highlighted + '</code></pre>';
};
var tableCss = 'table table-striped';
renderer.table = function (header, body) {
    return '<table class="' + tableCss + '">' + header + body + '</table>';
};

/*renderer.heading = function (text, level) {
    return '<h'+level+' data-category="">'+text+'</h'+level+'>';
};*/

var markedFilter = function (str) {
    str = marked(str.replace(/(<h[^>]*id\=[^>]*)>/g, function (s, s1) {
        if (s.indexOf('data-category') > -1) return s;
        return s1 + ' data-category=""' + '>';
    })).replace(/\<table\>/g, '<table class="' + tableCss + '">');
    return str;
};

var keywordHighlight = function(k, file){
    if(k==='') return file;
    var kws = k.split(' ');
    for(var i=0,l=kws.length;i<l;i++){
        var reg = new RegExp(kws[i].replace(/[\(\)\#\{\}\?\*\+]/g, '.'), 'g');
        file = file.replace(reg, function(s){
            return '<font class="kw">'+s+'</font>';
        });
    }
    return file;
};

marked.setOptions({
    renderer: renderer
});

var formateRender = function(obj){
    obj.config = global.config;
    return obj;
};

exports.index = function (req, res) {
    var k = (req.query.k || '').trim();
    var kws = k.replace(/[ ]+/g, ' ').split(' ');
    gitdata.getList(function (list) {
        if (k === '') {
            res.render('index.html', formateRender({ list: list, k: k }));
            return;
        }
        var arr = [];
        for (var i = 0, l = (list || []).length; i < l; i++) {
{'fraction_non_alphanumeric': 0.2273552551946742, 'fraction_numerical': 0.0036312285656647167, 'mean_word_length': 2.9664, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3047768', 'n_tokens_mistral': 1674, 'n_tokens_neox': 1670, 'n_words': 384}