<commit_before>cluster = require 'cluster'
fs = require('fs')

NUM_WORKERS = 1

if cluster.isMaster
	workerCount = 0

	cluster.on 'fork', (worker) ->
		console.log 'Worker', worker.id, "PID: ", worker.process.pid
		workerCount++

	cluster.on 'exit', (worker, code, signal) ->
		console.log 'worker died'
		workerCount--

	cluster.on 'listening', (worker) ->
		console.log 'worker', worker.id, "started listening"

	pidfile = __dirname + '/pids/master.pid'
	if fs.existsSync pidfile
		try
			oldMasterPid = fs.readFileSync pidfile
			process.kill oldMasterPid
			console.log "killing old master. PID: #{oldMasterPid}"
		catch
			# do nothing

	fs.open pidfile, 'w', (err, fd) ->
		throw "Unable to create master pid file: #{err}" if err?
		fs.write fd, process.pid

	checkWorkerCount = ->
		cluster.fork() if workerCount < NUM_WORKERS<commit_msg>let the os kernel load balance between workers<commit_after>cluster = require 'cluster'
fs = require('fs')

NUM_WORKERS = 1

# Let the OS do load balancing between node workers
cluster.schedulingPolicy = cluster.SCHED_NONE

if cluster.isMaster
	workerCount = 0

	cluster.on 'fork', (worker) ->
		console.log 'Worker', worker.id, "PID: ", worker.process.pid
		workerCount++

	cluster.on 'exit', (worker, code, signal) ->
		console.log 'worker died'
		workerCount--

	cluster.on 'listening', (worker) ->
		console.log 'worker', worker.id, "started listening"

	pidfile = __dirname + '/pids/master.pid'
	if fs.existsSync pidfile
		try
			oldMasterPid = fs.readFileSync pidfile
			process.kill oldMasterPid
			console.log "killing old master. PID: #{oldMasterPid}"
		catch
			# do nothing

	fs.open pidfile, 'w', (err, fd) ->
		throw "Unable to create master pid file: #{err}" if err?
		fs.write fd, process.pid

	checkWorkerCount = ->
		cluster.fork() if workerCount < NUM_WORKERS
{'fraction_non_alphanumeric': 0.12733260153677278, 'fraction_numerical': 0.0021953896816684962, 'mean_word_length': 3.900537634408602, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6365268', 'n_tokens_mistral': 663, 'n_tokens_neox': 598, 'n_words': 179}