<commit_before>require('/jiebaserver/scripts/main.js');
require("jsdom").env("", function(err, window) {
    if (err) {
        console.error(err);
        return;
    }
   var $ = require('jquery')(window);
});

//var express = require('express');
var GENERAL_DICT = require('/jiebaserver/scripts/data/dictionary.js');

var request = require("request");


var sub_array=[];     //for loop把temp_array裡的sub_array取出來依序丟到linked data proxy進行check
var check_result_array=[];  //各個sub_array進行check後回傳的check_result_array
var sub_result;         //各個check_result_array轉為string

//var URL = "http://exp-linked-data-proxy-2017.dlll.nccu.edu.tw/check/wiki,moedict,cbdb,tgaz,cdict,pixabay/" ;
var URL = "http://pc.pulipuli.info:3000/check_post/wiki,moedict,cbdb,tgaz,cdict,pixabay/";
var fs = require('fs');

var DIR_LJL='/jiebaserver/ljlarticle';

/*var _modules = ["wiki","moedict","cbdb","tgaz","pixabay"];*/
//app=express();

var bodyParser = require('body-parser');
app.use(bodyParser.json()); // support json encoded bodies
app.use(bodyParser.urlencoded({ extended: true })); // support encoded bodies

require("./session.js");
Cookies = require( "cookies" );

require('./database.js');
//--------------------------------------------------------------------------------------------
app.get("/client/js/linked-data-proxy-lib.js", function (_req, _res) {
    fs.readFile("/jiebaserver/client_js/linked-data-proxy-lib.js", 'utf8', function (err, data) {
        _res.setHeader('content-type', 'text/javascript');
        _res.send(data);
    });
});

app.get("/client/css/style.css", function (_req, _res) {
    fs.readFile("/jiebaserver/client_css/style.css", 'utf8', function (err, data) {
        _res.setHeader('content-type', 'text/css');
        _res.send(data);
    });
});
app.get("/client/css/tooltipster.bundle.min.css", function (_req, _res) {
    fs.readFile("/jiebaserver/client_css/tooltipster.bundle.min.css", 'utf8', function (err, data) {
        _res.setHeader('content-type', 'text/css');
        _res.send(data);
    });
});
app.get("/client/css/tooltipster-sideTip-noir.min.css", function (_req, _res) {
    fs.readFile("/jiebaserver/client_css/tooltipster-sideTip-noir.min.css", 'utf8', function (err, data) {
        _res.setHeader('content-type', 'text/css');
        _res.send(data);
    });
});

app.get("/client/js/jquery.js", function (_req, _res) {
    fs.readFile("/jiebaserver/client_js/jquery.js", 'utf8', function (err, data) {
        _res.setHeader('content-type', 'text/javascript');
        _res.send(data);
    });
});
app.get("/client/js/rangy-core.js", function (_req, _res) {
    fs.readFile("/jiebaserver/client_js/rangy-core.js", 'utf8', function (err, data) {
        _res.setHeader('content-type', 'text/javascript');
        _res.send(data);
    });
});
app.get("/client/js/utils.js", function (_req, _res) {
    fs.readFile("/jiebaserver/client_js/utils.js", 'utf8', function (err, data) {
        _res.setHeader('content-type', 'text/javascript');
        _res.send(data);
    });
});
app.get("/client/js/tooltipster.bundle.min.js", function (_req, _res) {
    fs.readFile("/jiebaserver/client_js/tooltipster.bundle.min.js", 'utf8', function (err, data) {
        _res.setHeader('content-type', 'text/javascript');
        _res.send(data);
    });
});

app.get("/client/js/loading.gif", function (_req, _res) {
    fs.readFile("/jiebaserver/client_js/loading.gif", function (err, data) {
        _res.setHeader('content-type', 'image/gif');
        _res.send(data);
    });
});

//----------------------------------------------------------------------------------------------
app.get("/article",function(req, res){
	fs.readFile("/jiebaserver/template/article.html", 'utf8',function(err, data){
		res.setHeader('content-type', 'text/html');

		//var _title = "title";
		
		//console.log(typeof(data));
		//console.log(data);
		fs.readFile('./ljlarticle/'+req.query.file, 'utf8', function (err,content) {
		  if (err) {
		    return console.log(err);
		  }
		  var _title = req.query.file;
		  var _content = content;

		  _content = _content.split("\n").join("<br />");

		  data = data.replace(/{{TITLE}}/g, _title);
		  data = data.replace(/{{CONTENT}}/g, _content);
		  res.send(data);
		});
		
		
	});
});


app.get("/directory",function(req, res){
	fs.readFile("/jiebaserver/template/directory.html", 'utf8',function(err, data){
		res.setHeader('content-type', 'text/html');
		

		fs.readdir(DIR_LJL, function(err, files) {
		    if (err) return;
		    var _content = "";
		    var _precontent = "";

		    var _directory_abstract_max_length = 20;
		    files.forEach(function(f) {
		    	var predata = fs.readFileSync("./ljlarticle/" + f, 'utf8');
		    	predata = predata.replace(/(?:\\[rnt]|[\r\n\t　]+)+/g, "").trim();
				//console.log(predata.substring(0, 19));
				if (predata.length > _directory_abstract_max_length) {
					_precontent = predata.substring(0, _directory_abstract_max_length) + "...";
				}
	    		_content = _content
	        		+ '<li><a href="article?file=' + f + '">' + f + '</a>: '+ _precontent +'</li>';	        
		    });
			data = data.replace(/{{CONTENT}}/g, _content);
			res.send(data);  
		});
	});
});
//----------------------------------------------------------------------------------------------

app.post("/parse_article", function (req, res) {

	var cookies = new Cookies( req, res );

	var article = req.body.article;
	//console.log(article);
	
	tableArticleCache.findOrCreate({
		where:{article:article}
	})
		.spread(function(articlecache, created) {

	  	// 2. 把暫存檔案的路徑放入COOKIE	
	  	var cache_id = articlecache.get('id');
	  	cookies.set("cache_id", cache_id);
	  	//console.log(created);
	  	//console.log(articlecache.get("result"));
	  	//console.log(articlecache.get({plain: true}));
	  	res.send("");
		if (created === true 
			|| articlecache.get("result") === null 
			|| articlecache.get("result") === "" ) {
		  	// 3. 開始斷詞或其他的處理
		  	_process(article, function(result) {
		  		// 4. 處理完之後放入暫存檔案 
		  		//console.log("4. 處理完之後放入暫存檔案 ");
		  		//console.log(result);
		  		tableArticleCache.update(
		  			{result:result},
		  			{where:{id:cache_id}}
		  		);
		  	});
	  	}
	  });	
});

app.get("/parse_article", function (req, res) {

	// 1. 取得COOKIE
	var cache_id;
	if (typeof(req.query.cache_id) === "undefined" ) {
		var cookies = new Cookies( req, res );
	    cache_id = cookies.get("cache_id");
	    
	}
	else {
		cache_id = req.query.cache_id;
	}
	cache_id = parseInt(cache_id, 10);
		


	// 2. 取得暫存檔案
	tableArticleCache
	  .findById(cache_id)
	  .then(function(articlecache) {
	  	if (articlecache === null || articlecache.get("result") === null || articlecache.get("result") === "") {
	  		// 3-1. IF 暫存檔案沒有資料: 回傳undefined
	  		res.jsonp(cache_id);
	  	}
	  	else {
	  		// 3-2. if 暫存檔案有資料
			// 回傳資料
			res.jsonp({
				result: articlecache.get("result"),
				cache_id: cache_id
			});
	  	}
	  });
});

app.get("/add_term",function (req, res){
	var term = req.query.term;
	var cache_id = req.query.cache_id;
	var dict_string="";
	// 1. 取得dict_custom.js
	var dict_custom='/jiebaserver/scripts/data/dict_custom.json';
	var foo=fs.openSync(dict_custom,'r+');
	fs.readFile(dict_custom, "utf-8" ,function(err,data){
		if(err){
			console.log('檔案讀取錯誤');
		}
		else{
			dict_string=data.toString();
			//console.log(dict_string);
			var term_existat = dict_string.indexOf('"'+term+'",');
			
	// 2. 檢查裡面有沒有這個term
	// 3. 如果沒有
			if(term_existat === -1){    
				//console.log("這是新詞");

			// 4. 把詞庫加入term，重組dict_custom.js
				var _head = dict_string.substr(0,2);
				var _new_term = '\t["' + term  + '", 99999, "n"],\n';
				var _foot = dict_string.substring(2,dict_string.length);

				var _new_dict = _head + _new_term + _foot;

				//var termtojson='["'+term+'", 999, "n" ]';
				// 	["郁離子",9999,"n"],
				fs.writeSync(foo,_new_dict,0);
				fs.close(foo);
				_load_custom_dict();

			// 5. 刪除快取 by cache id
			
			/*	tableArticleCache
				.findById(cache_id)
				.then(function(task){
					console.log(cache_id);
					return task.destroy();
				});
			*/
				tableArticleCache.update(
		  			{result:null},
		  			{where:{id:cache_id}}
		  		);
		  		res.jsonp(true);
			}
	// 6. 如果已經有這個詞了...
			else{
				//console.log("詞庫的第:"+term_existat+"個字找到 "+term+" 這個詞");
				

			}	
		}
		
	});
	
	//console.log(term);
});
// --------------------------------------------------------


var _custom_dict = undefined;
var _load_custom_dict = function () {
	var json = JSON.parse(fs.readFileSync('/jiebaserver/scripts/data/dict_custom.json', 'utf8'));
	_custom_dict=json;
	node_jieba_reset();
};

_load_custom_dict();
/**
 * callback(result)
 */
var _process = function (article, callback) {

	//callback("aaaaaaa12121212a" + article);
	//return;


var _parse_check_result_array = function (sub_array, check_result_array) {
		var _result = [];
		//console.log(sub_array);
		if (check_result_array !== undefined) {
			for (var i = 0; i < sub_array.length; i++ ) {
				if (sub_array[i] === "\n" && _replace_br === true) {<commit_msg>0506 還是不要整合在一起好了<commit_after>require('./scripts/main.js');
require("jsdom").env("", function(err, window) {
    if (err) {
        console.error(err);
        return;
    }
   var $ = require('jquery')(window);
});

//var express = require('express');
var GENERAL_DICT = require('./scripts/data/dictionary.js');

var request = require("request");


var sub_array=[];     //for loop把temp_array裡的sub_array取出來依序丟到linked data proxy進行check
var check_result_array=[];  //各個sub_array進行check後回傳的check_result_array
var sub_result;         //各個check_result_array轉為string

//var URL = "http://exp-linked-data-proxy-2017.dlll.nccu.edu.tw/check/wiki,moedict,cbdb,tgaz,cdict,pixabay/" ;
var URL = "http://pc.pulipuli.info:3000/check_post/wiki,moedict,cbdb,tgaz,cdict,pixabay/";
var fs = require('fs');

var DIR_LJL='../jiebaserver/ljlarticle';

/*var _modules = ["wiki","moedict","cbdb","tgaz","pixabay"];*/
//app=express();

var bodyParser = require('body-parser');
app.use(bodyParser.json()); // support json encoded bodies
app.use(bodyParser.urlencoded({ extended: true })); // support encoded bodies

require("./session.js");
Cookies = require( "cookies" );

require('./database.js');
//--------------------------------------------------------------------------------------------
app.get("/client/js/linked-data-proxy-lib.js", function (_req, _res) {
    fs.readFile("./client_js/linked-data-proxy-lib.js", 'utf8', function (err, data) {
        _res.setHeader('content-type', 'text/javascript');
        _res.send(data);
    });
});

app.get("/client/css/style.css", function (_req, _res) {
    fs.readFile("./client_css/style.css", 'utf8', function (err, data) {
        _res.setHeader('content-type', 'text/css');
        _res.send(data);
    });
});
app.get("/client/css/tooltipster.bundle.min.css", function (_req, _res) {
    fs.readFile("./client_css/tooltipster.bundle.min.css", 'utf8', function (err, data) {
        _res.setHeader('content-type', 'text/css');
        _res.send(data);
    });
});
app.get("/client/css/tooltipster-sideTip-noir.min.css", function (_req, _res) {
    fs.readFile("./client_css/tooltipster-sideTip-noir.min.css", 'utf8', function (err, data) {
        _res.setHeader('content-type', 'text/css');
        _res.send(data);
    });
});

app.get("/client/js/jquery.js", function (_req, _res) {
    fs.readFile("./client_js/jquery.js", 'utf8', function (err, data) {
        _res.setHeader('content-type', 'text/javascript');
        _res.send(data);
    });
});
app.get("/client/js/rangy-core.js", function (_req, _res) {
    fs.readFile("./client_js/rangy-core.js", 'utf8', function (err, data) {
        _res.setHeader('content-type', 'text/javascript');
        _res.send(data);
    });
});
app.get("/client/js/utils.js", function (_req, _res) {
    fs.readFile("./client_js/utils.js", 'utf8', function (err, data) {
        _res.setHeader('content-type', 'text/javascript');
        _res.send(data);
    });
});
app.get("/client/js/tooltipster.bundle.min.js", function (_req, _res) {
    fs.readFile("./client_js/tooltipster.bundle.min.js", 'utf8', function (err, data) {
        _res.setHeader('content-type', 'text/javascript');
        _res.send(data);
    });
});

app.get("/client/js/loading.gif", function (_req, _res) {
    fs.readFile("./client_js/loading.gif", function (err, data) {
        _res.setHeader('content-type', 'image/gif');
        _res.send(data);
    });
});

//----------------------------------------------------------------------------------------------
app.get("/article",function(req, res){
	fs.readFile("./template/article.html", 'utf8',function(err, data){
		res.setHeader('content-type', 'text/html');

		//var _title = "title";
		
		//console.log(typeof(data));
		//console.log(data);
		fs.readFile('./ljlarticle/'+req.query.file, 'utf8', function (err,content) {
		  if (err) {
		    return console.log(err);
		  }
		  var _title = req.query.file;
		  var _content = content;

		  _content = _content.split("\n").join("<br />");

		  data = data.replace(/{{TITLE}}/g, _title);
		  data = data.replace(/{{CONTENT}}/g, _content);
		  res.send(data);
		});
		
		
	});
});


app.get("/directory",function(req, res){
	fs.readFile("./template/directory.html", 'utf8',function(err, data){
		res.setHeader('content-type', 'text/html');
		

		fs.readdir(DIR_LJL, function(err, files) {
		    if (err) return;
		    var _content = "";
		    var _precontent = "";

		    var _directory_abstract_max_length = 20;
		    files.forEach(function(f) {
		    	var predata = fs.readFileSync("./ljlarticle/" + f, 'utf8');
		    	predata = predata.replace(/(?:\\[rnt]|[\r\n\t　]+)+/g, "").trim();
				//console.log(predata.substring(0, 19));
				if (predata.length > _directory_abstract_max_length) {
					_precontent = predata.substring(0, _directory_abstract_max_length) + "...";
				}
	    		_content = _content
	        		+ '<li><a href="article?file=' + f + '">' + f + '</a>: '+ _precontent +'</li>';	        
		    });
			data = data.replace(/{{CONTENT}}/g, _content);
			res.send(data);  
		});
	});
});
//----------------------------------------------------------------------------------------------

app.post("/parse_article", function (req, res) {

	var cookies = new Cookies( req, res );

	var article = req.body.article;
	//console.log(article);
	
	tableArticleCache.findOrCreate({
		where:{article:article}
	})
		.spread(function(articlecache, created) {

	  	// 2. 把暫存檔案的路徑放入COOKIE	
	  	var cache_id = articlecache.get('id');
	  	cookies.set("cache_id", cache_id);
	  	//console.log(created);
	  	//console.log(articlecache.get("result"));
	  	//console.log(articlecache.get({plain: true}));
	  	res.send("");
		if (created === true 
			|| articlecache.get("result") === null 
			|| articlecache.get("result") === "" ) {
		  	// 3. 開始斷詞或其他的處理
		  	_process(article, function(result) {
		  		// 4. 處理完之後放入暫存檔案 
		  		//console.log("4. 處理完之後放入暫存檔案 ");
		  		//console.log(result);
		  		tableArticleCache.update(
		  			{result:result},
		  			{where:{id:cache_id}}
		  		);
		  	});
	  	}
	  });	
});

app.get("/parse_article", function (req, res) {

	// 1. 取得COOKIE
	var cache_id;
	if (typeof(req.query.cache_id) === "undefined" ) {
		var cookies = new Cookies( req, res );
	    cache_id = cookies.get("cache_id");
	    
	}
	else {
		cache_id = req.query.cache_id;
	}
	cache_id = parseInt(cache_id, 10);
		


	// 2. 取得暫存檔案
	tableArticleCache
	  .findById(cache_id)
	  .then(function(articlecache) {
	  	if (articlecache === null || articlecache.get("result") === null || articlecache.get("result") === "") {
	  		// 3-1. IF 暫存檔案沒有資料: 回傳undefined
	  		res.jsonp(cache_id);
	  	}
	  	else {
	  		// 3-2. if 暫存檔案有資料
			// 回傳資料
			res.jsonp({
				result: articlecache.get("result"),
				cache_id: cache_id
			});
	  	}
	  });
});

app.get("/add_term",function (req, res){
	var term = req.query.term;
	var cache_id = req.query.cache_id;
	var dict_string="";
	// 1. 取得dict_custom.js
	var dict_custom='./scripts/data/dict_custom.json';
	var foo=fs.openSync(dict_custom,'r+');
	fs.readFile(dict_custom, "utf-8" ,function(err,data){
		if(err){
			console.log('檔案讀取錯誤');
		}
		else{
			dict_string=data.toString();
			//console.log(dict_string);
			var term_existat = dict_string.indexOf('"'+term+'",');
			
	// 2. 檢查裡面有沒有這個term
	// 3. 如果沒有
			if(term_existat === -1){    
				//console.log("這是新詞");

			// 4. 把詞庫加入term，重組dict_custom.js
				var _head = dict_string.substr(0,2);
				var _new_term = '\t["' + term  + '", 99999, "n"],\n';
				var _foot = dict_string.substring(2,dict_string.length);

				var _new_dict = _head + _new_term + _foot;

				//var termtojson='["'+term+'", 999, "n" ]';
				// 	["郁離子",9999,"n"],
				fs.writeSync(foo,_new_dict,0);
				fs.close(foo);
				_load_custom_dict();

			// 5. 刪除快取 by cache id
			
			/*	tableArticleCache
				.findById(cache_id)
				.then(function(task){
					console.log(cache_id);
					return task.destroy();
				});
			*/
				tableArticleCache.update(
		  			{result:null},
		  			{where:{id:cache_id}}
		  		);
		  		res.jsonp(true);
			}
	// 6. 如果已經有這個詞了...
			else{
				//console.log("詞庫的第:"+term_existat+"個字找到 "+term+" 這個詞");
				

			}	
		}
		
	});
	
	//console.log(term);
});
// --------------------------------------------------------


var _custom_dict = undefined;
var _load_custom_dict = function () {
    var json = JSON.parse(fs.readFileSync('./jiebaserver/scripts/data/dict_custom.json', 'utf8'));
    _custom_dict=json;
    node_jieba_reset();
};

_load_custom_dict();
/**
 * callback(result)
 */
var _process = function (article, callback) {

	//callback("aaaaaaa12121212a" + article);
	//return;


var _parse_check_result_array = function (sub_array, check_result_array) {
		var _result = [];
		//console.log(sub_array);
		if (check_result_array !== undefined) {
			for (var i = 0; i < sub_array.length; i++ ) {
				if (sub_array[i] === "\n" && _replace_br === true) {
{'fraction_non_alphanumeric': 0.22895604088700217, 'fraction_numerical': 0.008266770932245992, 'mean_word_length': 2.7740303541315345, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '470456', 'n_tokens_mistral': 7281, 'n_tokens_neox': 6700, 'n_words': 973}