<commit_before>"use strict"

var config = require("config")
var utils = require("../lib/utils.js")
var errorLib = require("../lib/error.js")
var db = require("../lib/db.js")
var clc = require('cli-color')
var async = require("async")
var sanitizeHtml = require('sanitize-html')
require("string_score")



exports.prepareAgents = function(cb){
	db.prepareAgents(function(){
		if (cb) cb()
	})
}


//turns on a process to keep the agent queue populated from the database
exports.populateShadowcatAgentsBuildQueue = function(cb){

	exports.shadowCatAgentsQueue = []
	var queueTimer = null
	var resumeOn = false


	console.log("Connecting to shadowcat...")

	db.returnCollectionShadowcat("bib",function(err,shadowcatBib){

		console.log("Starting Cursor...")

		// _id: 16547127
		var cursor = shadowcatBib.find({ }, { 'sc:agents' : 1, 'sc:research' : 1 }).batchSize(100)

		queueTimer = setInterval(function(){
			console.log("CHECK",exports.shadowCatAgentsQueue.length,resumeOn)
			if (exports.shadowCatAgentsQueue.length<5000){
				if (!resumeOn){
					cursor.resume()
					resumeOn = true
				}					
			}else{
				if (resumeOn){
					cursor.pause()
					resumeOn = false
				}					
				//the first run?
				if (cb){
					cb()
					cb = null
				}
			}
		},500)


		cursor.on('data', function(doc) {
			if (doc['sc:research']){
				exports.shadowCatAgentsQueue.push({bnumber: doc._id, agents: doc['sc:agents']})
			}
		})

		cursor.once('end', function() {		
			setTimeout(function(){

				clearTimeout(queueTimer)
				exports.shadowCatAgentsQueue.push(null)

				//console.log("populateShadowcatAgentsViaf - Done!")
				//cb(++agentId)				
				//agentsDatabase.close()
			},5000)
		})



	})

}


<commit_msg>queue problems<commit_after>"use strict"

var config = require("config")
var utils = require("../lib/utils.js")
var errorLib = require("../lib/error.js")
var db = require("../lib/db.js")
var clc = require('cli-color')
var async = require("async")
var sanitizeHtml = require('sanitize-html')
require("string_score")



exports.prepareAgents = function(cb){
	db.prepareAgents(function(){
		if (cb) cb()
	})
}


//turns on a process to keep the agent queue populated from the database
exports.populateShadowcatAgentsBuildQueue = function(cb){

	exports.shadowCatAgentsQueue = []
	var queueTimer = null
	var resumeOn = false


	console.log("Connecting to shadowcat...")

	db.returnCollectionShadowcat("bib",function(err,shadowcatBib){

		console.log("Starting Cursor...")

		// _id: 16547127
		var cursor = shadowcatBib.find({ }, { 'sc:agents' : 1, 'sc:research' : 1 }).batchSize(100)

		queueTimer = setInterval(function(){

			if (exports.shadowCatAgentsQueue.length<5000){
				if (!resumeOn){
					console.log("HERE")
					cursor.resume()
					resumeOn = true
				}					
			}else{
				if (resumeOn){
					cursor.pause()
					resumeOn = false
				}					
				//the first run?
				if (cb){
					cb()
					cb = null
				}
			}
			//wtf, hack.
			if (exports.shadowCatAgentsQueue.length>10000){
				cursor.pause()
				resumeOn = false
			}
		},1000)


		cursor.on('data', function(doc) {
			if (doc['sc:research']){
				exports.shadowCatAgentsQueue.push({bnumber: doc._id, agents: doc['sc:agents']})
			}
		})

		cursor.once('end', function() {		
			setTimeout(function(){

				clearTimeout(queueTimer)
				exports.shadowCatAgentsQueue.push(null)

				//console.log("populateShadowcatAgentsViaf - Done!")
				//cb(++agentId)				
				//agentsDatabase.close()
			},5000)
		})



	})

}



{'fraction_non_alphanumeric': 0.1596445973058183, 'fraction_numerical': 0.015477214101461736, 'mean_word_length': 3.3570536828963795, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4922808', 'n_tokens_mistral': 1470, 'n_tokens_neox': 1169, 'n_words': 230}