<commit_before>"use strict"

var cluster = require('cluster')
var serialize = require("../lib/serialize_catalog_terms_utils.js")
var serializeGeneral = require("../lib/serialize_utils.js")
var errorLib = require("../lib/error.js")



if (cluster.isMaster) {

	
	var countBibRecords = 0, countTotal = 0

	console.log("Initializing shadowcat TERMS Queue")
	
	serialize.prepareTerms(function(){

		serialize.populateShadowcatTermsBuildQueue(function(){
			
				//console.log(serialize.shadowCatTermsQueue[0])

				var spawnTimer = setInterval(function(){
					if (Object.keys(cluster.workers).length==1){
						clearInterval(spawnTimer)
					}else{

						var worker = cluster.fork()

						worker.on('message', function(msg) {


							if (serialize.shadowCatTermsQueue[0]===null){
								console.log("Sendiing QUIT msg",Object.keys(cluster.workers).length)
								//that is it, we've reached the end
								worker.send({ quit: true })

								setInterval(function(){
									if (Object.keys(cluster.workers).length<2){
										console.log("Finished Working records.")
										console.log("Catalog Terms | countBibRecords: " + countBibRecords + " countTotal: " + countTotal)
										console.log("Catalog Terms | countBibRecords: " + countBibRecords + " countTotal: " + countTotal)
										process.exit()
									}
								},1000)
								return false
							}



							if (msg.req) {

								//they are asking for new work							
								if (serialize.shadowCatTermsQueue.length>0){
									
									var workItem = serialize.shadowCatTermsQueue.splice(0,1)

									worker.send({ req: workItem })
								}else{
									console.log("Nothing left to work in the queue!")
									worker.send({ sleep: true })
								}
							}

							if (msg.countBibRecords) {
								countBibRecords++
							}
							if (msg.countTotal) {
								countTotal++
							}

							process.stdout.clearLine()
							process.stdout.cursorTo(0)
							process.stdout.write("Catalog Terms | countBibRecords: " + countBibRecords + " countTotal: " + countTotal )

							msg = null<commit_msg>catalog terms test2  #11<commit_after>"use strict"

var cluster = require('cluster')
var serialize = require("../lib/serialize_catalog_terms_utils.js")
var serializeGeneral = require("../lib/serialize_utils.js")
var errorLib = require("../lib/error.js")



if (cluster.isMaster) {

	
	var countBibRecords = 0, countTotal = 0

	console.log("Initializing shadowcat TERMS Queue")
	
	serialize.prepareTerms(function(){

		serialize.populateShadowcatTermsBuildQueue(function(){
			
				//console.log(serialize.shadowCatTermsQueue[0])

				var spawnTimer = setInterval(function(){
					if (Object.keys(cluster.workers).length==1){
						clearInterval(spawnTimer)
					}else{

						var worker = cluster.fork()

						worker.on('message', function(msg) {


							if (serialize.shadowCatTermsQueue[0]===null){
								console.log("Sendiing QUIT msg",Object.keys(cluster.workers).length)
								//that is it, we've reached the end
								worker.send({ quit: true })

								setInterval(function(){
									if (Object.keys(cluster.workers).length<2){
										console.log("Finished Working records.")
										console.log("Catalog Terms | countBibRecords: " + countBibRecords + " countTotal: " + countTotal)
										console.log("Catalog Terms | countBibRecords: " + countBibRecords + " countTotal: " + countTotal)
										process.exit()
									}
								},1000)
								return false
							}



							if (msg.req) {

								//they are asking for new work							
								if (serialize.shadowCatTermsQueue.length>0){
									
									var workItem = serialize.shadowCatTermsQueue.splice(0,1)
									console.log(workItem)
									worker.send({ req: workItem })
								}else{
									console.log("Nothing left to work in the queue!")
									worker.send({ sleep: true })
								}
							}

							if (msg.countBibRecords) {
								countBibRecords++
							}
							if (msg.countTotal) {
								countTotal++
							}

							process.stdout.clearLine()
							process.stdout.cursorTo(0)
							process.stdout.write("Catalog Terms | countBibRecords: " + countBibRecords + " countTotal: " + countTotal )

							msg = null
{'fraction_non_alphanumeric': 0.12473322267014465, 'fraction_numerical': 0.007351197533791795, 'mean_word_length': 2.5415617128463475, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5509794', 'n_tokens_mistral': 1856, 'n_tokens_neox': 1216, 'n_words': 243}