<commit_before>
var DEBUG = {
    enable_cache: true
};

/*var _modules = ["wiki","moedict","cbdb","tgaz","pixabay"];*/
//var URL = "http://exp-linked-data-proxy-2017.dlll.nccu.edu.tw/check/wiki,moedict,cbdb,tgaz,cdict,pixabay/" ; 
//var URL = "http://pc.pulipuli.info:3000/check_post/wiki,moedict,cbdb,tgaz,cdict,pixabay/";
//var URL = "http://localhost:3000/check/wiki,moedict,cbdb,tgaz,cdict,pixabay/" ;
//var URL = "http://localhost:3000/check_post/wiki,moedict/" ;	// 給NodeJS本機端用
var URL = "http://localhost:3000/check/wiki,moedict,cbdb,tgaz,cdict/";	// 給NodeJS本機端用

// -----------------------

//var sub_array = [];     //for loop把temp_array裡的sub_array取出來依序丟到linked data proxy進行check
//var check_result_array = [];  //各個sub_array進行check後回傳的check_result_array
//var sub_result;         //各個check_result_array轉為string

var GENERAL_DICT = require('../scripts/data/dictionary.js');
var _replace_br = false;

var stopword = ".,\/#!$%\^&\*;:{}=\-_`~()"
        + "«»/•† ‡¿№×‰ºª‱¶§|‖¦©℗®℠™¤​฿​​¢​₡​₢​$​₫​₯​​₠​€​ƒ​₣​​​₭​​₾​ℳ​₥​₦​₧​₱​₰​£​៛​​​₪​৳​​₮​₩​¥⁂❧☞‽◊⁀"
        + "\n\r。，<br>─「」；：:！？{}()[].0123456789"
        + "〇零一二三四五六七八九十月日時分"
        + "…、%％【】※『』 ㈠㈡　，、。．？！～＄％＠＆＃＊‧；︰…‥﹐﹒˙·﹔﹕‘’“”〝〞‵′〃├─┼┴┬┤┌┐╞═╪╡│▕└┘╭╮╰╯╔╦╗╠═╬╣╓╥╖╒╤╕║╚╩╝╟╫╢╙╨╜╞╪╡╘╧╛﹣﹦≡｜∣∥–︱—︳╴¯￣﹉﹊﹍﹎﹋﹌﹏︴﹨∕╲╱＼／↑↓←→↖↗↙↘〔〕【】﹝﹞〈〉﹙﹚《》（）｛｝﹛﹜『』「」＜＞≦≧﹤﹥︵︶︷︸︹︺︻︼︽︾︿﹀∩∪﹁﹂﹃﹄◎⊕⊙○●△▲▽▼☆★◇◆□■☎☏◐◑♡♥♣♧☻☺♠♤▪▫∴∵☜☞♫♬◊♦►◁∈∋◘◙♀♂♩♪☼￥〒￠￡※↔↕♨卍◈§♭＿ˍ▁▂▃▄▅▆▇█▏▎▍▌▋▊▉◢◣◥◤►◄▣▤▥▦▧▨▩▒░㊣㊟㊕㊗㊡㊝①②③④⑤⑥⑦⑧⑨⑩⑪⑫⑬⑭⑮⑯⑰⑱⑲⑳⑴⑵⑶⑷⑸⑹⑺⑻⑼⑽㈠㈡㈢㈣㈤㈥㈦㈧㈨㈩１２３４５６７８９０〡〢〣〤〥〦〧〨〩十卄卅ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫⓐⓑⓒⓓⓔⓕⓖⓗⓘⓙⓚⓛⓜⓝⓞⓟⓠⓡⓢⓣⓤⓥⓦⓧⓨⓩ⒜⒝⒞⒟⒠⒡⒢⒣⒤⒥⒦⒧⒨⒩⒪⒫⒬⒭⒮⒯⒰⒱⒲⒳⒴⒵ΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩανξοπρστυφχψωクラピカマニアチェックあなたのテスト≠〃てたこれまでキャンプシーズにかけるかがきいレャズンをえたコンパやすうなどすそビジネられていアウドアやにびせだも来つリビをしたとにオはわゅうごりよろしくそうまムペㄱㄲㄳㄴㄵㄶㄷㄸㄹㄺㄻㄼㄽㄾㄿㅀㅁㅂㅃㅄㅅㅆㅇㅈㅉㅊㅋㅌㅍㅎㅏㅐㅑㅒㅓㅔㅕㅖㅗㅘㅙㅚㅛㅜㅝㅞㅟㅠㅡㅢㅥㅦㅧㅨㅩㅪㅫㅬㅭㅮㅯㅰㅱㅲㅳㅴㅵㅶㅷㅸㅹㅺㅻㅼㅽㅾㅿㆀㆁㆂㆃㆄㆅㆆㆇㆈㆉㆊ╳＋﹢－×÷＝≠≒∞ˇ±√⊥∠∟⊿㏒㏑∫∮∵∴ㄅㄆㄇㄈㄉㄊㄋㄌㄍㄎㄏㄐㄑㄒㄓㄔㄕㄖㄗㄘㄙㄚㄛㄜㄝㄞㄟㄠㄡㄢㄣㄤㄥㄦㄧㄨㄩ˙ˊˇˋÄÃÇÊËÎÏÐÑÕÖÛÜãäêëîïõöŸŴŽŤŘŇĩħąčĤ";



// ------------------------

/**
 * @author moon
 * 接收來自client的資料
 */
app.post("/parse_article", function (req, res) {

    var cookies = new Cookies(req, res);

    var article = req.body.article;

    //console.log(article);
    _write_log(article);

    tableArticleCache.findOrCreate({
        where: {article: article}
    })
            .spread(function (articlecache, created) {

                // 2. 把暫存檔案的路徑放入COOKIE	
                var cache_id = articlecache.get('id');
                cookies.set("cache_id", cache_id);
                //console.log(created);
                //console.log(articlecache.get("result"));
                //console.log(articlecache.get({plain: true}));
                res.send("");
                if (created === true
                        || articlecache.get("result") === null
                        || articlecache.get("result") === "") {
                    // 3. 開始斷詞或其他的處理
                    _process(article, function (result) {
                        // 4. 處理完之後放入暫存檔案 
                        //console.log("4. 處理完之後放入暫存檔案 ");
                        //console.log(result);
                        tableArticleCache.update(
                                {result: result},
                        {where: {id: cache_id}}
                        );
                    });
                }
            });
});

// -------------------------------------------

app.get("/parse_article", function (req, res) {

    // 1. 取得COOKIE
    var cache_id;
    if (typeof (req.query.cache_id) === "undefined") {
        var cookies = new Cookies(req, res);
        cache_id = cookies.get("cache_id");

    }
    else {
        cache_id = req.query.cache_id;
    }
    cache_id = parseInt(cache_id, 10);

    //console.log(cache_id);

    // 2. 取得暫存檔案
    tableArticleCache
            .findById(cache_id)
            .then(function (articlecache) {
                //console.log("ok");
                if (articlecache === null || articlecache.get("result") === null || articlecache.get("result") === "") {
                    // 3-1. IF 暫存檔案沒有資料: 回傳undefined
                    console.log(["client端請求， 沒資料 ID: ", cache_id]);
                    res.jsonp(cache_id);
                }
                else {
                    // 3-2. if 暫存檔案有資料
                    // 回傳資料
                    var _result = articlecache.get("result");
                    
                    var _r = _result;
                    if (_r.length > 100) {
                        _r = _r.substr(0, 100) + "...";
                    }
                    console.log(["完成" + _r]);

                    if (DEBUG.enable_cache === false) {
                        articlecache.destroy({force: true});
                    }

                    res.jsonp({
                        result: _result,
                        cache_id: cache_id
                    });
                }
            });
});


/**
 * 處理斷詞
 * callback(result)
 */
var _process = function (article, callback) {

    //callback("aaaaaaa12121212a" + article);
    //return;

    // ----------------------------------------

    

    // end of var _node_jieba_parsing_callback = function (_result) {
    // --------------------------


    //article = article.substr(0, 50);

    //article=article.replace(/\"/g, "");
    //article=article.replace(/(?:\\[rnt]|[\r\n\t]+)+/g, "");

    //console.log(article);
    //return;

    //console.log(_custom_dict);
    if (article.indexOf("<br>") > -1) {
        article = article.replace(/<br>/g, "\n");
        _replace_br = true;
    }

    node_jieba_parsing([GENERAL_DICT, _custom_dict], article, function (_result) {
        _node_jieba_parsing_callback(_result, callback);
    });	// end of node_jieba_parsing([dict1, dict2], article, function (_result) {
}; // end of proces: var _process = function (article, callback) {

// --------------------------------------------

REQUEST_COUNT = 0;
REQUEST_COUNT_MAX = 10;

var _node_jieba_parsing_callback = function (_result, callback) {
    //console.log(_result);
    //return;

    var temp_array = [];    //把斷完詞的array以每50個詞進行切分  切分為數個array ex:[[a,b,....],[c,d,....]]
    var joined_result = "";    //把每個sub_result結合起來 準備回傳給client

    //console.log(article);
    var BATCH_CHECK = 100;

    for (var t = 0, len = _result.length; t < len; t += BATCH_CHECK) {
        temp_array.push(_result.slice(t, t + BATCH_CHECK));
    }

    // array: temp_array
    // limit: temp_array.length
    // callback: callback(joined_result);
    
    var _loop = function (_i) {
        if (_i < temp_array.length) {
            _do_loop(_i);
        }
        else {
            // 結束了
            //console.log(joined_result);
            callback(joined_result);
        }
    };	// var _loop = function (_i) {

    var _do_loop = function (_i) {
        if (REQUEST_COUNT > REQUEST_COUNT_MAX) {
            setTimeout(function () {
                console.log(["等待中...", _i, temp_array.length, temp_array[_i][0], REQUEST_COUNT]);
                _do_loop(_i);
            }, 1000 * getRandomArbitrary(1,20));
            return;
        }
        
        //console.log("送出第" + _i + "次");
        // 執行迴圈
        var sub_array = [];
        var _send_array = [];
        for (var _t = 0; _t < temp_array[_i].length; _t++) {
            //var _term = temp_array[_i][_t].replace(/(?:\\[rnt]|[\r\n\t]+)+/g, "").trim();
            var _term = temp_array[_i][_t];
            //if (_term !== " " 
            //&& _term !== "" 
            //&& _term !== "　"
            //&& _term !== "\t"<commit_msg>0528 防止送出太多訊息<commit_after>
var DEBUG = {
    enable_cache: true
};

/*var _modules = ["wiki","moedict","cbdb","tgaz","pixabay"];*/
//var URL = "http://exp-linked-data-proxy-2017.dlll.nccu.edu.tw/check/wiki,moedict,cbdb,tgaz,cdict,pixabay/" ; 
//var URL = "http://pc.pulipuli.info:3000/check_post/wiki,moedict,cbdb,tgaz,cdict,pixabay/";
//var URL = "http://localhost:3000/check/wiki,moedict,cbdb,tgaz,cdict,pixabay/" ;
//var URL = "http://localhost:3000/check_post/wiki,moedict/" ;	// 給NodeJS本機端用
var URL = "http://localhost:3000/check/wiki,moedict,cbdb,tgaz,cdict/";	// 給NodeJS本機端用

// -----------------------

//var sub_array = [];     //for loop把temp_array裡的sub_array取出來依序丟到linked data proxy進行check
//var check_result_array = [];  //各個sub_array進行check後回傳的check_result_array
//var sub_result;         //各個check_result_array轉為string

var GENERAL_DICT = require('../scripts/data/dictionary.js');
var _replace_br = false;

var stopword = ".,\/#!$%\^&\*;:{}=\-_`~()"
        + "«»/•† ‡¿№×‰ºª‱¶§|‖¦©℗®℠™¤​฿​​¢​₡​₢​$​₫​₯​​₠​€​ƒ​₣​​​₭​​₾​ℳ​₥​₦​₧​₱​₰​£​៛​​​₪​৳​​₮​₩​¥⁂❧☞‽◊⁀"
        + "\n\r。，<br>─「」；：:！？{}()[].0123456789"
        + "〇零一二三四五六七八九十月日時分"
        + "…、%％【】※『』 ㈠㈡　，、。．？！～＄％＠＆＃＊‧；︰…‥﹐﹒˙·﹔﹕‘’“”〝〞‵′〃├─┼┴┬┤┌┐╞═╪╡│▕└┘╭╮╰╯╔╦╗╠═╬╣╓╥╖╒╤╕║╚╩╝╟╫╢╙╨╜╞╪╡╘╧╛﹣﹦≡｜∣∥–︱—︳╴¯￣﹉﹊﹍﹎﹋﹌﹏︴﹨∕╲╱＼／↑↓←→↖↗↙↘〔〕【】﹝﹞〈〉﹙﹚《》（）｛｝﹛﹜『』「」＜＞≦≧﹤﹥︵︶︷︸︹︺︻︼︽︾︿﹀∩∪﹁﹂﹃﹄◎⊕⊙○●△▲▽▼☆★◇◆□■☎☏◐◑♡♥♣♧☻☺♠♤▪▫∴∵☜☞♫♬◊♦►◁∈∋◘◙♀♂♩♪☼￥〒￠￡※↔↕♨卍◈§♭＿ˍ▁▂▃▄▅▆▇█▏▎▍▌▋▊▉◢◣◥◤►◄▣▤▥▦▧▨▩▒░㊣㊟㊕㊗㊡㊝①②③④⑤⑥⑦⑧⑨⑩⑪⑫⑬⑭⑮⑯⑰⑱⑲⑳⑴⑵⑶⑷⑸⑹⑺⑻⑼⑽㈠㈡㈢㈣㈤㈥㈦㈧㈨㈩１２３４５６７８９０〡〢〣〤〥〦〧〨〩十卄卅ⅠⅡⅢⅣⅤⅥⅦⅧⅨⅩⅪⅫⓐⓑⓒⓓⓔⓕⓖⓗⓘⓙⓚⓛⓜⓝⓞⓟⓠⓡⓢⓣⓤⓥⓦⓧⓨⓩ⒜⒝⒞⒟⒠⒡⒢⒣⒤⒥⒦⒧⒨⒩⒪⒫⒬⒭⒮⒯⒰⒱⒲⒳⒴⒵ΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩανξοπρστυφχψωクラピカマニアチェックあなたのテスト≠〃てたこれまでキャンプシーズにかけるかがきいレャズンをえたコンパやすうなどすそビジネられていアウドアやにびせだも来つリビをしたとにオはわゅうごりよろしくそうまムペㄱㄲㄳㄴㄵㄶㄷㄸㄹㄺㄻㄼㄽㄾㄿㅀㅁㅂㅃㅄㅅㅆㅇㅈㅉㅊㅋㅌㅍㅎㅏㅐㅑㅒㅓㅔㅕㅖㅗㅘㅙㅚㅛㅜㅝㅞㅟㅠㅡㅢㅥㅦㅧㅨㅩㅪㅫㅬㅭㅮㅯㅰㅱㅲㅳㅴㅵㅶㅷㅸㅹㅺㅻㅼㅽㅾㅿㆀㆁㆂㆃㆄㆅㆆㆇㆈㆉㆊ╳＋﹢－×÷＝≠≒∞ˇ±√⊥∠∟⊿㏒㏑∫∮∵∴ㄅㄆㄇㄈㄉㄊㄋㄌㄍㄎㄏㄐㄑㄒㄓㄔㄕㄖㄗㄘㄙㄚㄛㄜㄝㄞㄟㄠㄡㄢㄣㄤㄥㄦㄧㄨㄩ˙ˊˇˋÄÃÇÊËÎÏÐÑÕÖÛÜãäêëîïõöŸŴŽŤŘŇĩħąčĤ";



// ------------------------

/**
 * @author moon
 * 接收來自client的資料
 */
app.post("/parse_article", function (req, res) {

    var cookies = new Cookies(req, res);

    var article = req.body.article;

    //console.log(article);
    _write_log(article);

    tableArticleCache.findOrCreate({
        where: {article: article}
    })
            .spread(function (articlecache, created) {

                // 2. 把暫存檔案的路徑放入COOKIE	
                var cache_id = articlecache.get('id');
                cookies.set("cache_id", cache_id);
                //console.log(created);
                //console.log(articlecache.get("result"));
                //console.log(articlecache.get({plain: true}));
                res.send("");
                if (created === true
                        || articlecache.get("result") === null
                        || articlecache.get("result") === "") {
                    // 3. 開始斷詞或其他的處理
                    _process(article, function (result) {
                        // 4. 處理完之後放入暫存檔案 
                        //console.log("4. 處理完之後放入暫存檔案 ");
                        //console.log(result);
                        tableArticleCache.update(
                                {result: result},
                        {where: {id: cache_id}}
                        );
                    });
                }
            });
});

// -------------------------------------------

app.get("/parse_article", function (req, res) {

    // 1. 取得COOKIE
    var cache_id;
    if (typeof (req.query.cache_id) === "undefined") {
        var cookies = new Cookies(req, res);
        cache_id = cookies.get("cache_id");

    }
    else {
        cache_id = req.query.cache_id;
    }
    cache_id = parseInt(cache_id, 10);

    //console.log(cache_id);

    // 2. 取得暫存檔案
    tableArticleCache
            .findById(cache_id)
            .then(function (articlecache) {
                //console.log("ok");
                if (articlecache === null || articlecache.get("result") === null || articlecache.get("result") === "") {
                    // 3-1. IF 暫存檔案沒有資料: 回傳undefined
                    console.log(["client端請求， 沒資料 ID: ", cache_id]);
                    res.jsonp(cache_id);
                }
                else {
                    // 3-2. if 暫存檔案有資料
                    // 回傳資料
                    var _result = articlecache.get("result");
                    
                    var _r = _result;
                    if (_r.length > 100) {
                        _r = _r.substr(0, 100) + "...";
                    }
                    console.log(["完成" + _r]);

                    if (DEBUG.enable_cache === false) {
                        articlecache.destroy({force: true});
                    }

                    res.jsonp({
                        result: _result,
                        cache_id: cache_id
                    });
                }
            });
});


/**
 * 處理斷詞
 * callback(result)
 */
var _process = function (article, callback) {

    //callback("aaaaaaa12121212a" + article);
    //return;

    // ----------------------------------------

    

    // end of var _node_jieba_parsing_callback = function (_result) {
    // --------------------------


    //article = article.substr(0, 50);

    //article=article.replace(/\"/g, "");
    //article=article.replace(/(?:\\[rnt]|[\r\n\t]+)+/g, "");

    //console.log(article);
    //return;

    //console.log(_custom_dict);
    if (article.indexOf("<br>") > -1) {
        article = article.replace(/<br>/g, "\n");
        _replace_br = true;
    }

    node_jieba_parsing([GENERAL_DICT, _custom_dict], article, function (_result) {
        _node_jieba_parsing_callback(_result, callback);
    });	// end of node_jieba_parsing([dict1, dict2], article, function (_result) {
}; // end of proces: var _process = function (article, callback) {

// --------------------------------------------

REQUEST_COUNT = 0;
REQUEST_COUNT_MAX = 10;

var _node_jieba_parsing_callback = function (_result, callback) {
    //console.log(_result);
    //return;

    var temp_array = [];    //把斷完詞的array以每50個詞進行切分  切分為數個array ex:[[a,b,....],[c,d,....]]
    var joined_result = "";    //把每個sub_result結合起來 準備回傳給client

    //console.log(article);
    var BATCH_CHECK = 100;

    for (var t = 0, len = _result.length; t < len; t += BATCH_CHECK) {
        temp_array.push(_result.slice(t, t + BATCH_CHECK));
    }

    // array: temp_array
    // limit: temp_array.length
    // callback: callback(joined_result);
    
    var _loop = function (_i) {
        if (_i < temp_array.length) {
            _do_loop(_i);
        }
        else {
            // 結束了
            //console.log(joined_result);
            callback(joined_result);
        }
    };	// var _loop = function (_i) {

    var _do_loop = function (_i) {
        if (REQUEST_COUNT > REQUEST_COUNT_MAX) {
            setTimeout(function () {
                //console.log(["等待中...", _i, temp_array.length, temp_array[_i][0], REQUEST_COUNT]);
                _do_loop(_i);
            }, 1000 * getRandomArbitrary(1,20));
            return;
        }
        
        //console.log("送出第" + _i + "次");
        // 執行迴圈
        var sub_array = [];
        var _send_array = [];
        for (var _t = 0; _t < temp_array[_i].length; _t++) {
            //var _term = temp_array[_i][_t].replace(/(?:\\[rnt]|[\r\n\t]+)+/g, "").trim();
            var _term = temp_array[_i][_t];
            //if (_term !== " " 
            //&& _term !== "" 
            //&& _term !== "　"
            //&& _term !== "\t"
{'fraction_non_alphanumeric': 0.22646822204344327, 'fraction_numerical': 0.01246983105390185, 'mean_word_length': 1.8155907889769725, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1188581', 'n_tokens_mistral': 8162, 'n_tokens_neox': 7842, 'n_words': 696}