<commit_before>
var config
var samples={
	allSamples:[],
	topSamples:[]
}
var fs=require('fs')
var path=require('path')
var fileCount=0

var crypto=require('crypto')

process.on('message',messageHandler)

var messageTypes={
	'init':init,
	'updateTestCase':updateTestCase,
	'samplesLow':generateNewTestCase
}

var config
function messageHandler(message){
	if(messageTypes.hasOwnProperty(message.type)){
		messageTypes[message.type](message.data)
	}
	else{
		console.log('No message type: '+message.type)
	}
}

/*
	List files from inputDirectory and sort by size.
*/
function init(conf){
	config=conf
	var allSamples=fs.readdirSync(config.inputDirectory).map(function(fileName){
	if(!fs.statSync(path.resolve(config.inputDirectory,fileName)).isDirectory()){
		return path.resolve(config.inputDirectory,fileName)
	}else{
		return undefined
	}
	});

	allSamples=allSamples.filter(function(file){if(file){return file}else{return false}})
	allSamples.sort(function(a,b){
		if(config.reverse)
			return fs.statSync(path.resolve(config.inputDirectory,b)).size-fs.statSync(path.resolve(config.inputDirectory,a)).size
		else
			return fs.statSync(path.resolve(config.inputDirectory,a)).size-fs.statSync(path.resolve(config.inputDirectory,b)).size
	})
	process.send({type:'initReady',data:{files:allSamples}})
}

/*
	updateTestCase-message handler
*/
function updateTestCase(message){
	if(message.action=='remove')
		removeTestCase(message.data)
	else if(message.action=='save')
		saveNewSamples(message.data)
	if(!message.data.noNew)
		generateNewTestCase()
	else
		newTestCase(undefined)
}


/*
	Save new sample to outputDirectory
*/
function saveNewSamples(data){
	var fileContent=fs.readFileSync(data.file)
	var original=path.resolve(data.file)
	var fileName=crypto.createHash('sha1').update(fileContent).digest('hex')
	var fullName=path.resolve(config.outputDirectory,path.basename(fileName)+path.extname(data.file))
	fs.writeFileSync(fullName,fileContent)
	samples.allSamples.push(fullName)
	samples.topSamples.push({file:fullName,weight:200})
	console.log('['+(new Date().getTime())+']File: '+fullName+' newblocks:'+data.newBlocks+' corpussize: '+samples.allSamples.length+' totalblocks: '+data.totalBlocks)									
	if( original != fullName){
		removeTestCase(data)
	}
}

/*
	Remove sample from lists
*/
function removeTestCase(data){
	var file=path.resolve(data.file)
	var inInputDir=(path.resolve(file).indexOf(config.inputDirectory)!=-1)
	var inOutputDir=(path.resolve(file).indexOf(config.outputDirectory)!=-1)
	if(!inInputDir && fs.existsSync(file)){
		if(inInputDir||inOutputDir){
			if(samples.allSamples.indexOf(file)!=-1){
				samples.allSamples.splice(samples.allSamples.indexOf(file),1)
			}
			for(var x=0; x<samples.topSamples.length; x++){
				if(samples.topSamples[x].file==file){
					samples.topSamples.splice(x,1)		
				}
			}
		}
		fs.unlink(file)
	}
}


function rint(max){
	return Math.floor(Math.random()*max)
}

function ra(arr){
	return arr[rint(arr.length)]
}

/*
	Select sample for surku and call surku to generate new test case.
*/
function generateNewTestCase(){
	if(samples.topSamples.length>1 && rint(3)){
		var sample=samples.topSamples.splice(rint(samples.topSamples.length),1)[0]
		var newWeight=sample.weight-1
		if(newWeight>0)
			samples.topSamples.push({file:sample.file,weight:newWeight})
		surkuFunction(sample.file,newTestCase)
	}
	else{
		surkuFunction(ra(samples.allSamples),newTestCase)
	}
}
var surkuConfig={
	maxMutations:20,
	minMutations:1,
	chunkSize:3000
}
var S=require('surku');
var surku=new S(surkuConfig)
/*
	Generate new test case and write it to file.
	TODO: To avoid extra readFileSync, maybe generate multiple test cases from same input-file.
*/
function surkuFunction(sampleFile,callback){
	if(sampleFile){
		var prefix=new Date().getTime()
		var fileName=config.tempDirectory+'/samples/'+prefix+fileCount+path.extname(sampleFile)
		var fileContent=fs.readFileSync(sampleFile)
		var chunkSize=Math.ceil(fileContent.length/100)
		if(chunkSize>3000){
			surku.config.chunkSize=3000
		}else if(chunkSize<500){
			surku.config.chunkSize=500<commit_msg>Without output file counting, a race condition can occurs in output file naming.<commit_after>
var config
var samples={
	allSamples:[],
	topSamples:[]
}
var fs=require('fs')
var path=require('path')
var fileCount=0

var crypto=require('crypto')

process.on('message',messageHandler)

var messageTypes={
	'init':init,
	'updateTestCase':updateTestCase,
	'samplesLow':generateNewTestCase
}

var config
function messageHandler(message){
	if(messageTypes.hasOwnProperty(message.type)){
		messageTypes[message.type](message.data)
	}
	else{
		console.log('No message type: '+message.type)
	}
}

/*
	List files from inputDirectory and sort by size.
*/
function init(conf){
	config=conf
	var allSamples=fs.readdirSync(config.inputDirectory).map(function(fileName){
	if(!fs.statSync(path.resolve(config.inputDirectory,fileName)).isDirectory()){
		return path.resolve(config.inputDirectory,fileName)
	}else{
		return undefined
	}
	});

	allSamples=allSamples.filter(function(file){if(file){return file}else{return false}})
	allSamples.sort(function(a,b){
		if(config.reverse)
			return fs.statSync(path.resolve(config.inputDirectory,b)).size-fs.statSync(path.resolve(config.inputDirectory,a)).size
		else
			return fs.statSync(path.resolve(config.inputDirectory,a)).size-fs.statSync(path.resolve(config.inputDirectory,b)).size
	})
	process.send({type:'initReady',data:{files:allSamples}})
}

/*
	updateTestCase-message handler
*/
function updateTestCase(message){
	if(message.action=='remove')
		removeTestCase(message.data)
	else if(message.action=='save')
		saveNewSamples(message.data)
	if(!message.data.noNew)
		generateNewTestCase()
	else
		newTestCase(undefined)
}


/*
	Save new sample to outputDirectory
*/
function saveNewSamples(data){
	var fileContent=fs.readFileSync(data.file)
	var original=path.resolve(data.file)
	var fileName=crypto.createHash('sha1').update(fileContent).digest('hex')
	var fullName=path.resolve(config.outputDirectory,path.basename(fileName)+path.extname(data.file))
	fs.writeFileSync(fullName,fileContent)
	samples.allSamples.push(fullName)
	samples.topSamples.push({file:fullName,weight:200})
	console.log('['+(new Date().getTime())+']File: '+fullName+' newblocks:'+data.newBlocks+' corpussize: '+samples.allSamples.length+' totalblocks: '+data.totalBlocks)									
	if( original != fullName){
		removeTestCase(data)
	}
}

/*
	Remove sample from lists
*/
function removeTestCase(data){
	var file=path.resolve(data.file)
	var inInputDir=(path.resolve(file).indexOf(config.inputDirectory)!=-1)
	var inOutputDir=(path.resolve(file).indexOf(config.outputDirectory)!=-1)
	if(!inInputDir && fs.existsSync(file)){
		if(inInputDir||inOutputDir){
			if(samples.allSamples.indexOf(file)!=-1){
				samples.allSamples.splice(samples.allSamples.indexOf(file),1)
			}
			for(var x=0; x<samples.topSamples.length; x++){
				if(samples.topSamples[x].file==file){
					samples.topSamples.splice(x,1)		
				}
			}
		}
		fs.unlink(file)
	}
}


function rint(max){
	return Math.floor(Math.random()*max)
}

function ra(arr){
	return arr[rint(arr.length)]
}

/*
	Select sample for surku and call surku to generate new test case.
*/
function generateNewTestCase(){
	if(samples.topSamples.length>1 && rint(3)){
		var sample=samples.topSamples.splice(rint(samples.topSamples.length),1)[0]
		var newWeight=sample.weight-1
		if(newWeight>0)
			samples.topSamples.push({file:sample.file,weight:newWeight})
		surkuFunction(sample.file,newTestCase)
	}
	else{
		surkuFunction(ra(samples.allSamples),newTestCase)
	}
}
var surkuConfig={
	maxMutations:20,
	minMutations:1,
	chunkSize:3000
}
var S=require('surku');
var surku=new S(surkuConfig)
/*
	Generate new test case and write it to file.
	TODO: To avoid extra readFileSync, maybe generate multiple test cases from same input-file.
*/
function surkuFunction(sampleFile,callback){
	if(sampleFile){
		var prefix=new Date().getTime()
		fileCount++
		var fileName=config.tempDirectory+'/samples/'+prefix+fileCount+path.extname(sampleFile)
		var fileContent=fs.readFileSync(sampleFile)
		var chunkSize=Math.ceil(fileContent.length/100)
		if(chunkSize>3000){
			surku.config.chunkSize=3000
		}else if(chunkSize<500){
			surku.config.chunkSize=500
{'fraction_non_alphanumeric': 0.14738873477264502, 'fraction_numerical': 0.009890242431552286, 'mean_word_length': 6.267309377738826, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5898902', 'n_tokens_mistral': 2995, 'n_tokens_neox': 2774, 'n_words': 434}