<commit_before>var etc=require("./etc.js"),
    msgtype=require("./msgtype.js");

function constructMessage(type,args){
	var len=6+args.map(function(a){return 4+a.length;}).reduce(function(a,b){return a+b;},0);
	var buf=new Buffer(len);
	//console.log("constructing message with len",len)
	buf.writeUInt32BE(len,0);
	buf.writeUInt8(type,4);
	buf.writeUInt8(args.length,5);
	var cursor=6;
	for(var i=0;i<args.length;i++){
		buf.writeUInt32BE(args[i].length,cursor);
		cursor+=4;
		args[i].copy(buf,cursor);
		cursor+=args[i].length;
	}
	//console.log("constructing message with len",len,"result:",buf);
	return buf;
}

function parseMessage(buf){
	var buflen=buf.length;
	if(buflen<4)return false;
	var len=buf.readUInt32BE(0);
	if(buflen<len)return false;
	console.log(buf.slice(0,len));
	var type=buf.readUInt8(4);
	var numargs=buf.readUInt8(5);
	var cursor=6;
	var args=new Array(numargs),arglen;
	for(var i=0;i<numargs;i++){
		//console.log("pM: i="+i);
		if(cursor+4>len)return {type:null,args:null,len:len};
		arglen=buf.readUInt32BE(cursor);
		cursor+=4;
		//console.log("pM: cursor="+cursor);
		if(cursor+arglen>len)return {type:null,args:null,len:len};
		args[i]=new Buffer(arglen);
		buf.copy(args[i],0,cursor,cursor+arglen);
		cursor+=arglen;
	}<commit_msg>filedelete fix<commit_after>var etc=require("./etc.js"),
    msgtype=require("./msgtype.js");

function constructMessage(type,args){
	var len=6+args.map(function(a){return 4+a.length;}).reduce(function(a,b){return a+b;},0);
	var buf=new Buffer(len);
	//console.log("constructing message with len",len)
	buf.writeUInt32BE(len,0);
	buf.writeUInt8(type,4);
	buf.writeUInt8(args.length,5);
	var cursor=6;
	for(var i=0;i<args.length;i++){
		if(!(args[i] instanceof Buffer))args[i]=new Buffer(""+args[i]);
		buf.writeUInt32BE(args[i].length,cursor);
		cursor+=4;
		args[i].copy(buf,cursor);
		cursor+=args[i].length;
	}
	//console.log("constructing message with len",len,"result:",buf);
	return buf;
}

function parseMessage(buf){
	var buflen=buf.length;
	if(buflen<4)return false;
	var len=buf.readUInt32BE(0);
	if(buflen<len)return false;
	console.log(buf.slice(0,len));
	var type=buf.readUInt8(4);
	var numargs=buf.readUInt8(5);
	var cursor=6;
	var args=new Array(numargs),arglen;
	for(var i=0;i<numargs;i++){
		//console.log("pM: i="+i);
		if(cursor+4>len)return {type:null,args:null,len:len};
		arglen=buf.readUInt32BE(cursor);
		cursor+=4;
		//console.log("pM: cursor="+cursor);
		if(cursor+arglen>len)return {type:null,args:null,len:len};
		args[i]=new Buffer(arglen);
		buf.copy(args[i],0,cursor,cursor+arglen);
		cursor+=arglen;
	}
{'fraction_non_alphanumeric': 0.2045367166474433, 'fraction_numerical': 0.023836985774702037, 'mean_word_length': 6.790419161676646, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1229145', 'n_tokens_mistral': 1100, 'n_tokens_neox': 1064, 'n_words': 143}