<commit_before>var WebKit = require('webkitgtk');
var debug = require('debug')('express-dom');

module.exports = Pool;


function Pool(opts, dom) {
	this.dom = dom;
	dom.navigator = WebKit.navigator;
	this.list = [];
	this.count = 0;
	this.max = opts.max || 8;
	this.notices = 1;
	this.queue = [];
	this.destroyTimeout = opts.destroyTimeout || 600000; // destroy the page if not used for that time
	this.idleTimeout = opts.idleTimeout || 180000; // unload the page if not used for that time
	setInterval(this.wipe.bind(this), this.idleTimeout / 4);
}

Pool.prototype.wipe = function() {
	var now = Date.now();
	var page;
	var nlist = [];
	for (var i=0; i < this.list.length; i++) {
		page = this.list[i];
		nlist.push(page);
		if (page.locked) continue;
		if (page.releaseTime) {
			if (now > page.releaseTime + this.destroyTimeout) {
				nlist.pop();
				page.destroy(error);
			}
		} else if (page.pingTime && now > page.pingTime + this.idleTimeout) {
			page.unload(error);
		}
	}
	// in case destroy calls have been made
	if (nlist.length != this.list.length) {
		this.list = nlist;
		this.count = this.list.length;
	}
};

Pool.prototype.acquire = function(cb) {
	var create = false;
	var page;
	if (this.count < this.max) {
		this.count++;
		create = true;
	} else {
		for (var i=0; i < this.list.length; i++) {
			page = this.list[i];
			if (!page.locked) {
				break;
			}
			page = null;
		}
	}

	if (page) {
		page.locked = true;
		delete page.releaseTime;
		page.reset(function(err) {
			page.pingTime = Date.now();
			cb(null, page);
		});
	} else if (create) {
		WebKit(this.dom.settings, function(err, page) {
			if (err) return cb(err);
			page.locked = true;
			this.list.push(page);
			page.prepare = page.preload; // just let us name it prepare
			cb(null, page);
		}.bind(this));
	} else {
		this.queue.push({ts: Date.now(), cb: cb});
		if (this.queue.length > (this.max * this.notices)) {
			this.notices++;
			console.info("express-dom", this.queue.length, "queued acquisitions - consider raising dom.pool.max above", this.max);
		}
	}
};

Pool.prototype.release = function(page) {<commit_msg>Destroy web pages after maxloads, better unload handling<commit_after>var WebKit = require('webkitgtk');
var debug = require('debug')('express-dom');

module.exports = Pool;


function Pool(opts, dom) {
	this.dom = dom;
	dom.navigator = WebKit.navigator;
	this.list = [];
	this.count = 0;
	this.max = opts.max || 8;
	this.maxloads = 100;
	this.notices = 1;
	this.queue = [];
	this.destroyTimeout = opts.destroyTimeout || 600000; // destroy the page if not used for that time
	this.idleTimeout = opts.idleTimeout || 180000; // unload the page if not used for that time
	setInterval(this.wipe.bind(this), this.idleTimeout / 4);
}

Pool.prototype.wipe = function() {
	var now = Date.now();
	var page;
	var nlist = [];
	for (var i=0; i < this.list.length; i++) {
		page = this.list[i];
		nlist.push(page);
		if (page.locked) continue;
		if (page.releaseTime) {
			if (now > page.releaseTime + this.destroyTimeout || page.numloads > this.maxloads) {
				nlist.pop();
				// prevents the page from being grabbed during unload/destroy
				page.locked = true;
				page.unload().catch(error).then(function() {
					return page.destroy();
				}).catch(error);
			}
		} else if (page.pingTime && now > page.pingTime + this.idleTimeout) {
			// ensures idling page is unloaded after some time
			page.locked = true;
			page.unload(function(err) {
				page.locked = false;
				error(err);
			});
		}
	}
	// in case destroy calls have been made
	if (nlist.length != this.list.length) {
		this.list = nlist;
		this.count = this.list.length;
	}
};

Pool.prototype.acquire = function(cb) {
	var create = false;
	var page;
	if (this.count < this.max) {
		this.count++;
		create = true;
	} else {
		for (var i=0; i < this.list.length; i++) {
			page = this.list[i];
			if (!page.locked) {
				break;
			}
			page = null;
		}
	}

	if (page) {
		page.locked = true;
		delete page.releaseTime;
		page.numloads = (page.numloads || 0) + 1;
		page.reset(function(err) {
			page.pingTime = Date.now();
			cb(null, page);
		});
	} else if (create) {
		WebKit(this.dom.settings, function(err, page) {
			if (err) return cb(err);
			page.locked = true;
			this.list.push(page);
			page.prepare = page.preload; // just let us name it prepare
			cb(null, page);
		}.bind(this));
	} else {
		this.queue.push({ts: Date.now(), cb: cb});
		if (this.queue.length > (this.max * this.notices)) {
			this.notices++;
			console.info("express-dom", this.queue.length, "queued acquisitions - consider raising dom.pool.max above", this.max);
		}
	}
};

Pool.prototype.release = function(page) {
{'fraction_non_alphanumeric': 0.1646655231560892, 'fraction_numerical': 0.00879073756432247, 'mean_word_length': 3.2255434782608696, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '45301', 'n_tokens_mistral': 1864, 'n_tokens_neox': 1684, 'n_words': 409}