<commit_before>'use strict'
var levenshtein = require('./levenshtein.js');
var nodejieba = require("nodejieba");
var pagerank = require('./pagerank.js');
var isummary = require('./isummary.js');
var stringSimilarity = require('string-similarity');

const summary_clear_rules = [
	[/^编者按：/,''],
	[/^[^，日天前]{1,4}，/,''],
	[/^“/,''],
	[/^其该此而[^，]{1,10}，/,''],
	[/^[并且至于而]+/,'']
]

class Summarizer{
	constructor(shingle,filter){
		this.shingle = shingle;
		this.filter = filter;
	}

	splitToParagraphs(content){
		return content.split('\n\n');
	}

	splitToSentences(content){
		return content.split(/(?:\n|\u3002|\uff1f|\uff01)/g);
	}

	splitToTerms(content){
		let self = this;
		let terms = [];
		let ps = Array.isArray(content)?content:self.splitToParagraphs(content);
		for(let x of ps){
			let sens = self.splitToSentences(x);
			let parag = [];
			for(let y of sens){
				let sentens = [];
				for(let z of nodejieba.cut(y.trim()))sentens.push(z);
			    if(sentens.length>0){
			    	if(sentens[sentens.length-1]!='。')sentens.push('。');
			    	parag.push(sentens);
			    }
			}
			if(parag.length>0)terms.push(parag);
		}
		return terms;
	}

	splitToWords(content){
		return nodejieba.cut(content);
	}

	getSimilarityGraph(sentences){
		var graph = new Array();
	    for(let s of sentences) {
	      let sentenceSimilarity = new Array();
	      for(let t of sentences) {
	        sentenceSimilarity.push(levenshtein.get(s, t));
	      }
	      graph.push(sentenceSimilarity);
	    }
	    return graph;
	}

	getTextRank(graph){
		return pagerank.Pagerank(graph, 0.85, 0.0001, function (err, res) {
	      if (err) throw new Error(err)
	    });
	}

	isValidateSentence(charArr){
		if(!charArr)return false;
		let slen=0,dlen = 0;  
	    for (let i=0; i<charArr.length; i++) {  
		     let c = charArr[i].charCodeAt();   
		     if ((c >= 0x0001 && c <= 0x007e) || (0xff60<=c && c<=0xff9f)) {   
		       	slen++;
		     }else {
		      	dlen++;   
		     }   
	    }
	    if(slen > dlen)return false;
	    let ok = true;
	    if(this.filter){
	    	let sentense = charArr.join('');
	    	for(let rule of this.filter){
	    		if(new RegExp(rule,'ig').test(sentense)){
	    			ok = false;
	    			break;
	    		}
	    	}
	    }
	    return ok;
	}

	score(sentence_rank, model_score, title_similarity, len_score){
		// return Math.pow(sentence_rank, 0.35) * Math.pow(model_score, 0.35) * Math.pow(title_similarity, 0.1) * Math.pow(len_score, 0.2);
		// return Math.pow(sentence_rank, 10) * Math.pow(model_score, 10)
		return Math.pow(sentence_rank, 1) * Math.pow(Math.min(model_score, 0.7), 10) * Math.pow(title_similarity, 1) * Math.pow(len_score, 1);
	}

	summarize(content, expected_length, title){
	    let terms = this.splitToTerms(content);
	    let cursor = 0;
	    let sentences = [];
	    let spaces_title = title ? nodejieba.cut(title.trim()).join(' ') : [];
	    for(let x = 0;x<terms.length;x++){
	    	for(let y = 0;y<terms[x].length;y++){
	    		if(this.shingle){
	    			let sentence = [];
		    		let cu_size = 0;
		    		let max_covered = 0;
		    		for(let z=y;z<terms[x].length;z++){
		    			cu_size += terms[x][z].length;
		    			if(z>y && cu_size>expected_length*1)break;
		    			else {sentence = sentence.concat(terms[x][z]);}
		    			max_covered = z;
		    		}
		    		sentences.push(sentence);
		    		if(max_covered >= terms[x].length-1)break;
	    		}else sentences.push(terms[x][y]);
	    	}
	    }<commit_msg>enhance score<commit_after>'use strict'
var levenshtein = require('./levenshtein.js');
var nodejieba = require("nodejieba");
var pagerank = require('./pagerank.js');
var isummary = require('./isummary.js');
var stringSimilarity = require('string-similarity');

const summary_clear_rules = [
	[/^编者按：/,''],
	[/^[^，日天前]{1,4}，/,''],
	[/^“/,''],
	[/^其该此而[^，]{1,10}，/,''],
	[/^[并且至于而]+/,'']
]

class Summarizer{
	constructor(shingle,filter){
		this.shingle = shingle;
		this.filter = filter;
	}

	splitToParagraphs(content){
		return content.split('\n\n');
	}

	splitToSentences(content){
		return content.split(/(?:\n|\u3002|\uff1f|\uff01)/g);
	}

	splitToTerms(content){
		let self = this;
		let terms = [];
		let ps = Array.isArray(content)?content:self.splitToParagraphs(content);
		for(let x of ps){
			let sens = self.splitToSentences(x);
			let parag = [];
			for(let y of sens){
				let sentens = [];
				for(let z of nodejieba.cut(y.trim()))sentens.push(z);
			    if(sentens.length>0){
			    	if(sentens[sentens.length-1]!='。')sentens.push('。');
			    	parag.push(sentens);
			    }
			}
			if(parag.length>0)terms.push(parag);
		}
		return terms;
	}

	splitToWords(content){
		return nodejieba.cut(content);
	}

	getSimilarityGraph(sentences){
		var graph = new Array();
	    for(let s of sentences) {
	      let sentenceSimilarity = new Array();
	      for(let t of sentences) {
	        sentenceSimilarity.push(levenshtein.get(s, t));
	      }
	      graph.push(sentenceSimilarity);
	    }
	    return graph;
	}

	getTextRank(graph){
		return pagerank.Pagerank(graph, 0.85, 0.0001, function (err, res) {
	      if (err) throw new Error(err)
	    });
	}

	isValidateSentence(charArr){
		if(!charArr)return false;
		let slen=0,dlen = 0;  
	    for (let i=0; i<charArr.length; i++) {  
		     let c = charArr[i].charCodeAt();   
		     if ((c >= 0x0001 && c <= 0x007e) || (0xff60<=c && c<=0xff9f)) {   
		       	slen++;
		     }else {
		      	dlen++;   
		     }   
	    }
	    if(slen > dlen)return false;
	    let ok = true;
	    if(this.filter){
	    	let sentense = charArr.join('');
	    	for(let rule of this.filter){
	    		if(new RegExp(rule,'ig').test(sentense)){
	    			ok = false;
	    			break;
	    		}
	    	}
	    }
	    return ok;
	}

	score(sentence_rank, model_score, title_similarity, len_score){
		// return Math.pow(sentence_rank, 0.35) * Math.pow(model_score, 0.35) * Math.pow(title_similarity, 0.1) * Math.pow(len_score, 0.2);
		// return Math.pow(sentence_rank, 10) * Math.pow(model_score, 10)
		console.log(sentence_rank, model_score, title_similarity, len_score);
		// return Math.pow(sentence_rank, 10) * Math.pow(Math.min(model_score, 0.7), 10) * Math.pow(title_similarity, 1) * Math.pow(len_score, 1);
		// let final_score = Math.ceil(sentence_rank* 1000)*100000 + Math.ceil(len_score*1000)*100 + Math.ceil(model_score*100) + Math.ceil(title_similarity * 10);
		let final_score = sentence_rank * 0.5 + len_score*0.2 + model_score*0.2 + title_similarity * 0.1;
		// console.log("score=>",final_score);
		return final_score
	}

	summarize(content, expected_length, title){
	    let terms = this.splitToTerms(content);
	    let cursor = 0;
	    let sentences = [];
	    let spaces_title = title ? nodejieba.cut(title.trim()).join(' ') : [];
	    for(let x = 0;x<terms.length;x++){
	    	for(let y = 0;y<terms[x].length;y++){
	    		if(this.shingle){
	    			let sentence = [];
		    		let cu_size = 0;
		    		let max_covered = 0;
		    		for(let z=y;z<terms[x].length;z++){
		    			cu_size += terms[x][z].length;
		    			if(z>y && cu_size>expected_length*1)break;
		    			else {sentence = sentence.concat(terms[x][z]);}
		    			max_covered = z;
		    		}
		    		sentences.push(sentence);
		    		if(max_covered >= terms[x].length-1)break;
	    		}else sentences.push(terms[x][y]);
	    	}
	    }
{'fraction_non_alphanumeric': 0.176147293212421, 'fraction_numerical': 0.022945864248419896, 'mean_word_length': 2.940985381700054, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7206099', 'n_tokens_mistral': 3200, 'n_tokens_neox': 2779, 'n_words': 478}