<commit_before>const argv = require("args-parser")(process.argv);
const webhook = require("./misc/webhookStatus");

const botHandler = require("./worker_handling/bot");
const siteHandler = require("./worker_handling/site");
function handleWorker(worker) {
	if(worker.type === "bot") botHandler(worker);
	else if(worker.type === "website") siteHandler(worker);
}

Object.defineProperty(cluster, "onlineWorkers", {
	get: () => Object.keys(cluster.workers)
		.map(id => cluster.workers[id])
		.filter(work => work.isConnected())
});

let totalShards = argv.shards || 1;
async function init() {
	await (require("./misc/rethink")).init();
	webhook({
		title: `Master started`,
		color: 0x0000FF,
		timestamp: new Date()
	});

	if(totalShards < 1 || isNaN(totalShards)) totalShards = 1;
	console.log(totalShards, argv.shards);
	statsd({ type: "gauge", stat: "shards", value: totalShards });

	let shardsPerWorker;
	if(argv.perWorker && argv.perWorker >= 1 && !isNaN(argv.perWorker)) {
		shardsPerWorker = argv.perWorker;
	} else {
		let coreCount = require("os").cpus().length;
		if(coreCount >= totalShards) shardsPerWorker = 1;
		else shardsPerWorker = Math.ceil(totalShards / coreCount);
	}

	const workerCount = Math.ceil(totalShards / shardsPerWorker);
	statsd({ type: "gauge", stat: "workers", value: workerCount });
	for(let i = 0; i < workerCount; i++) {
		let shardStart = i * shardsPerWorker, shardEnd = ((i + 1) * shardsPerWorker) - 1;
		if(shardEnd > totalShards - 1) shardEnd = totalShards - 1;
		let shardRange = shardStart === shardEnd ? `shard ${shardStart}` : `shards ${shardStart}-${shardEnd}`;

		const worker = cluster.fork();<commit_msg>argv debug<commit_after>const argv = require("args-parser")(process.argv);
const webhook = require("./misc/webhookStatus");

const botHandler = require("./worker_handling/bot");
const siteHandler = require("./worker_handling/site");
function handleWorker(worker) {
	if(worker.type === "bot") botHandler(worker);
	else if(worker.type === "website") siteHandler(worker);
}

Object.defineProperty(cluster, "onlineWorkers", {
	get: () => Object.keys(cluster.workers)
		.map(id => cluster.workers[id])
		.filter(work => work.isConnected())
});

let totalShards = argv.shards || 1;
async function init() {
	await (require("./misc/rethink")).init();
	webhook({
		title: `Master started`,
		color: 0x0000FF,
		timestamp: new Date()
	});

	if(totalShards < 1 || isNaN(totalShards)) totalShards = 1;
	console.log(argv, process.argv);
	statsd({ type: "gauge", stat: "shards", value: totalShards });

	let shardsPerWorker;
	if(argv.perWorker && argv.perWorker >= 1 && !isNaN(argv.perWorker)) {
		shardsPerWorker = argv.perWorker;
	} else {
		let coreCount = require("os").cpus().length;
		if(coreCount >= totalShards) shardsPerWorker = 1;
		else shardsPerWorker = Math.ceil(totalShards / coreCount);
	}

	const workerCount = Math.ceil(totalShards / shardsPerWorker);
	statsd({ type: "gauge", stat: "workers", value: workerCount });
	for(let i = 0; i < workerCount; i++) {
		let shardStart = i * shardsPerWorker, shardEnd = ((i + 1) * shardsPerWorker) - 1;
		if(shardEnd > totalShards - 1) shardEnd = totalShards - 1;
		let shardRange = shardStart === shardEnd ? `shard ${shardStart}` : `shards ${shardStart}-${shardEnd}`;

		const worker = cluster.fork();
{'fraction_non_alphanumeric': 0.17177541729893778, 'fraction_numerical': 0.009104704097116844, 'mean_word_length': 5.081180811808118, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '565536', 'n_tokens_mistral': 1191, 'n_tokens_neox': 1118, 'n_words': 248}