<commit_before>// JavaScript plugin for Bugzilla reporting.
// 
// Copyright 2006, <NAME>
// 

var BufferedReader    = Packages.java.io.BufferedReader;
var InputStreamReader = Packages.java.io.InputStreamReader;
var DataOutputStream  = Packages.java.io.DataOutputStream;
var URL               = Packages.java.net.URL;
var Socket            = Packages.java.net.Socket;


function log(msg) {
	dumpln("BUGZILLA [" + (new Date()) + "] " + msg);
}

String.prototype.trim =
function _trim() {
	return this.replace(/^\s+/, "").replace(/\s+$/, "");
}


// Constructor: Bugzilla
function Bugzilla(mods, irc) {
	this._mods = mods;
	this._irc = irc;
	this._debugChannel = "#testing42";
	
	this._targetList = new Object();
	this._seenMsgs = new Object();
	this._firstTime = true;
	
	var targets = this._mods.odb.retrieve(BugmailTarget, "");
	for (var i = 0; i < targets.size(); i++) {
		var target = targets.get(i);
		
		// Allow the feed to save itself when it makes changes.
		target.__mods = this._mods;
		target.save = function _target_save() { this.__mods.odb.update(this) };
		
		target.init(this);
		this._targetList[target.target] = target;
	}
	
	this._mods.interval.callBack("bugmail-check", 10000 /* 10s */, 1);
}


Bugzilla.prototype.info = [
		"Bugzilla bug notification plugin.",
		"<NAME>",
		"<EMAIL>",
		"$Rev$$Date$"
	];


Bugzilla.prototype.optionsGeneral = ["POP3Server", "POP3Port", "POP3Account", "POP3Password"];


// Callback for all intervals from this plugin.
Bugzilla.prototype.interval = function(param, mods, irc) {
	function mapCase(s, a, b) {
		return a.toUpperCase() + b.toLowerCase();
	};
	
	if (param) {
		var name = "_" + param.replace(/-(\w)(\w+)/g, mapCase) + "Interval";
		
		if (name in this) {
			try {
				this[name](param, mods, irc);
			} catch(ex) {
				log("Exception in " + name + ": " + ex + (ex.fileName ? " at <" + ex.fileName + ":" + ex.lineNumber + ">":""));
				irc.sendMessage(this._debugChannel, "Exception in " + name + ": " + ex + (ex.fileName ? " at <" + ex.fileName + ":" + ex.lineNumber + ">":""));
			}
		} else {
			irc.sendMessage(this._debugChannel, "Interval code missing: " + name);
		}
		
	} else {
		irc.sendMessage(this._debugChannel, "Unnamed interval attempted!");
	}
}


// Interval: bugmail-check
Bugzilla.prototype._bugmailCheckInterval = function(param, mods, irc) {
	var bugs = new Array();
	
	try {
		var self = this;
		this._checkMail(function(pop3, messageID) {
			if (!(messageID in self._seenMsgs)) {
				if (!this._firstTime) {
					var bug = new BugmailParser(pop3.getMessage(messageID));
					bugs.push(bug);
				}
				self._seenMsgs[messageID] = true;
			}
			return true;
		});
	} catch(ex) {
		log("Error checking bugmail: " + ex);
		this._mods.interval.callBack("bugmail-check", 30000 /* 30s */, 1);
		return;
	}
	
	this._mods.interval.callBack("bugmail-check", 30000 /* 30s */, 1);
	
	if ((bugs.length == 0) || this._firstTime) {
		this._firstTime = false;
	} else {
		this._spam(bugs);
	}
	
	this._updateActivityDB(bugs);
	this._updateFlagDB(bugs);
}


// Command: AddComponent
Bugzilla.prototype.commandAddComponent = function(mes, mods, irc) {
	var params = mods.util.getParams(mes, 2);
	if (params.size() <= 2) {
		irc.sendContextReply(mes, "Syntax: Bugzilla.AddComponent <channel> <component>");
		return;
	}
	var channel = String(params.get(1)).trim();
	
	var target = this._targetList[channel];
	if (!target) {
		target = this._targetList[channel] = new BugmailTarget(this, channel);
		this._mods.odb.save(target);
		target.__mods = this._mods;
		target.save = function _target_save() { this.__mods.odb.update(this) };
	}
	
	var component = String(params.get(2)).trim();
	
	if (target.addComponent(component)) {
		irc.sendContextReply(mes, "Channel '" + channel + "' will now receive bugmail for '" + component + "'.");
	} else {
		irc.sendContextReply(mes, "Channel '" + channel + "' already receives bugmail for '" + component + "'.");
	}
}
Bugzilla.prototype.commandAddComponent.help = [
		"Adds a new component to the list for a channel.",
		"<channel> <component>",
		"<channel> is the name of the channel to adjust",
		"<component> is the component to start sending bugmail for"
	];


// Command: RemoveComponent
Bugzilla.prototype.commandRemoveComponent = function(mes, mods, irc) {
	var params = mods.util.getParams(mes, 2);
	if (params.size() <= 2) {
		irc.sendContextReply(mes, "Syntax: Bugzilla.RemoveComponent <channel> <component>");
		return;
	}
	var channel = String(params.get(1)).trim();
	
	var target = this._targetList[channel];
	if (!target) {
		target = this._targetList[channel] = new BugmailTarget(this, channel);
		this._mods.odb.save(target);
		target.__mods = this._mods;
		target.save = function _target_save() { this.__mods.odb.update(this) };
	}
	
	var component = String(params.get(2)).trim();
	
	if (target.removeComponent(component)) {
		irc.sendContextReply(mes, "Channel '" + channel + "' will no longer receive bugmail for '" + component + "'.");
	} else {
		irc.sendContextReply(mes, "Channel '" + channel + "' doesn't receive bugmail for '" + component + "'.");
	}
}
Bugzilla.prototype.commandRemoveComponent.help = [
		"Removes a component from the list for a channel.",
		"<channel> <component>",
		"<channel> is the name of the channel to adjust",
		"<component> is the component to stop sending bugmail for"
	];


// Command: Log
Bugzilla.prototype.commandLog = function(mes, mods, irc) {
	var params = mods.util.getParams(mes, 1);
	if (params.size() <= 1) {
		irc.sendContextReply(mes, "Syntax: Bugzilla.Log <bug number>");
		return;
	}
	var bugNum = Number(params.get(1));
	
	var bugs = new Array();
	try {
		var self = this;
		this._checkMail(function(pop3, messageID) {
			var bug = new BugmailParser(pop3.getMessage(messageID));
			if ((bug.bugNumber == bugNum) || (bugNum == 0)) {
				bugs.push(bug);
			}
			return true;
		});
	} catch(ex) {
		irc.sendContextReply(mes, "Error checking bugmail: " + ex);
		return;
	}
	
	if (bugs.length == 0) {
		irc.sendContextReply(mes, "Nothing found in log for bug " + bugNum);
	} else {
		this._spam(bugs, mes);
	}
}
Bugzilla.prototype.commandLog.help = [
		"Shows all changes capture for a single bug.",
		"<bug number>",
		"<bug number> is the bug number to show the log of"
	];


// Command: Queue
Bugzilla.prototype.commandQueue = function(mes, mods, irc) {
	var params = mods.util.getParams(mes, 1);
	if (params.size() <= 1) {
		irc.sendContextReply(mes, "Syntax: Bugzilla.Queue <email>");
		return;
	}
	var email = String(params.get(1)).trim();
	
	var flags = this._mods.odb.retrieve(BugzillaSavedFlagRequest, "WHERE `from` = \"" + this._mods.odb.escapeString(email) + "\"");
	var list = new Array();
	
	for (var i = 0; i < flags.size(); i++) {
		var f = flags.get(i);
		list.push("bug " + f.bug + " (" + f.name + ")");
	}
	
	if (list.length == 0) {
		irc.sendContextReply(mes, "Queue is empty.");
	} else {
		irc.sendContextReply(mes, "Queue: " + list.join(", ") + ".");
	}
}
Bugzilla.prototype.commandQueue.help = [
		"Shows the review queue for a user, as seen by the bot.",
		"<email>",
		"<email> is the user to show the queue of"
	];


// Command: RebuildDB
Bugzilla.prototype.commandRebuildDB = function(mes, mods, irc) {
	var params = mods.util.getParams(mes, 0);
	if (params.size() <= 0) {
		irc.sendContextReply(mes, "Syntax: Bugzilla.RebuildDB");
		return;
	}
	
	// Delete all saved flags.
	var flags = this._mods.odb.retrieve(BugzillaSavedFlagRequest, "");
	for (var i = 0; i < flags.size(); i++)
		this._mods.odb["delete"](flags.get(i));
	
	var bugs = new Array();
	try {
		var self = this;
		this._checkMail(function(pop3, messageID) {
			var bug = new BugmailParser(pop3.getMessage(messageID));
			bugs.push(bug);
			return true;
		});
	} catch(ex) {
		irc.sendContextReply(mes, "Error checking bugmail: " + ex);
		return;
	}
	
	this._updateActivityDB(bugs);
	this._updateFlagDB(bugs);
	irc.sendContextReply(mes, "Database rebuilt from bugmail.");
}
Bugzilla.prototype.commandRebuildDB.help = [
		"Rebuilds the list of requested flags.",
		""
	];


// Internal: calls back for each message.
Bugzilla.prototype._checkMail = function(callbackFn) {
	var pop3host     = this._mods.plugin.callAPI("Options", "GetGeneralOption", ["POP3Server",   ""]);
	var pop3port     = this._mods.plugin.callAPI("Options", "GetGeneralOption", ["POP3Port",     "110"]);
	var pop3account  = this._mods.plugin.callAPI("Options", "GetGeneralOption", ["POP3Account",  ""]);
	var pop3password = this._mods.plugin.callAPI("Options", "GetGeneralOption", ["POP3Password", ""]);
	
	var pop3 = new POP3Server(pop3host, pop3port, pop3account, pop3password);
	var messageIDList = pop3.getMessageList();
	for (var i = 0; i < messageIDList.length; i++) {
		if (!callbackFn(pop3, messageIDList[i])) {
			break;
		}
	}
	pop3.close();
}


// Internal: spam list of bugs at their appropriate channels.
Bugzilla.prototype._spam = function(bugs, mes) {
	for (var i = 0; i < bugs.length; i++) {
		var things = new Array();
		
		if (bugs[i].isNew) {
			things.push("filed by " + bugs[i].from);
		}
		
		if (bugs[i].newAttachment) {
			var msgp = "added attachment " + bugs[i].newAttachment.number;
			if (bugs[i].newAttachment.label) {
				msgp += " (" + bugs[i].newAttachment.label + ")";
			}
			things.push(msgp);
		}
		
		if (bugs[i].changes.length > 0) {
			var resolved = -1;
			var fixed    = -1;
			var resolution = "";
			for (var j = 0; j < bugs[i].changes.length; j++) {
				if ((bugs[i].changes[j].name == "Status") && (bugs[i].changes[j].newValue == "RESOLVED"))
					resolved = j;
				if ((bugs[i].changes[j].name == "Resolution") && bugs[i].changes[j].newValue) {
					fixed = j;
					resolution = bugs[i].changes[j].newValue;
				}
			}
			
			if ((resolved >= 0) && (fixed >= 0)) {
				things.push("resolved " + resolution);
			}
			
			var list = new Array();
			for (var j = 0; j < bugs[i].changes.length; j++) {
				if ((resolved >= 0) && (fixed >= 0) && ((resolved == j) || (fixed == j))) {
					continue;
				}
				list.push(bugs[i].changes[j].name
						+ (bugs[i].changes[j].oldValue ? " from '" + bugs[i].changes[j].oldValue + "'" : "")
						+ " to '" + bugs[i].changes[j].newValue + "'");
			}
			if (list.length > 0) {
				things.push("changed " + list.join(", "));
			}
		}
		
		if (bugs[i].removed.length > 0) {
			var list = new Array();
			for (var j = 0; j < bugs[i].removed.length; j++) {
				list.push(bugs[i].removed[j].name + " '" + bugs[i].removed[j].value + "'");
			}<commit_msg>Add reopened messages, and improve resolved handling code.<commit_after>// JavaScript plugin for Bugzilla reporting.
// 
// Copyright 2006, <NAME>
// 

var BufferedReader    = Packages.java.io.BufferedReader;
var InputStreamReader = Packages.java.io.InputStreamReader;
var DataOutputStream  = Packages.java.io.DataOutputStream;
var URL               = Packages.java.net.URL;
var Socket            = Packages.java.net.Socket;


function log(msg) {
	dumpln("BUGZILLA [" + (new Date()) + "] " + msg);
}

String.prototype.trim =
function _trim() {
	return this.replace(/^\s+/, "").replace(/\s+$/, "");
}


// Constructor: Bugzilla
function Bugzilla(mods, irc) {
	this._mods = mods;
	this._irc = irc;
	this._debugChannel = "#testing42";
	
	this._targetList = new Object();
	this._seenMsgs = new Object();
	this._firstTime = true;
	
	var targets = this._mods.odb.retrieve(BugmailTarget, "");
	for (var i = 0; i < targets.size(); i++) {
		var target = targets.get(i);
		
		// Allow the feed to save itself when it makes changes.
		target.__mods = this._mods;
		target.save = function _target_save() { this.__mods.odb.update(this) };
		
		target.init(this);
		this._targetList[target.target] = target;
	}
	
	this._mods.interval.callBack("bugmail-check", 10000 /* 10s */, 1);
}


Bugzilla.prototype.info = [
		"Bugzilla bug notification plugin.",
		"<NAME>",
		"<EMAIL>",
		"$Rev$$Date$"
	];


Bugzilla.prototype.optionsGeneral = ["POP3Server", "POP3Port", "POP3Account", "POP3Password"];


// Callback for all intervals from this plugin.
Bugzilla.prototype.interval = function(param, mods, irc) {
	function mapCase(s, a, b) {
		return a.toUpperCase() + b.toLowerCase();
	};
	
	if (param) {
		var name = "_" + param.replace(/-(\w)(\w+)/g, mapCase) + "Interval";
		
		if (name in this) {
			try {
				this[name](param, mods, irc);
			} catch(ex) {
				log("Exception in " + name + ": " + ex + (ex.fileName ? " at <" + ex.fileName + ":" + ex.lineNumber + ">":""));
				irc.sendMessage(this._debugChannel, "Exception in " + name + ": " + ex + (ex.fileName ? " at <" + ex.fileName + ":" + ex.lineNumber + ">":""));
			}
		} else {
			irc.sendMessage(this._debugChannel, "Interval code missing: " + name);
		}
		
	} else {
		irc.sendMessage(this._debugChannel, "Unnamed interval attempted!");
	}
}


// Interval: bugmail-check
Bugzilla.prototype._bugmailCheckInterval = function(param, mods, irc) {
	var bugs = new Array();
	
	try {
		var self = this;
		this._checkMail(function(pop3, messageID) {
			if (!(messageID in self._seenMsgs)) {
				if (!this._firstTime) {
					var bug = new BugmailParser(pop3.getMessage(messageID));
					bugs.push(bug);
				}
				self._seenMsgs[messageID] = true;
			}
			return true;
		});
	} catch(ex) {
		log("Error checking bugmail: " + ex);
		this._mods.interval.callBack("bugmail-check", 30000 /* 30s */, 1);
		return;
	}
	
	this._mods.interval.callBack("bugmail-check", 30000 /* 30s */, 1);
	
	if ((bugs.length == 0) || this._firstTime) {
		this._firstTime = false;
	} else {
		this._spam(bugs);
	}
	
	this._updateActivityDB(bugs);
	this._updateFlagDB(bugs);
}


// Command: AddComponent
Bugzilla.prototype.commandAddComponent = function(mes, mods, irc) {
	var params = mods.util.getParams(mes, 2);
	if (params.size() <= 2) {
		irc.sendContextReply(mes, "Syntax: Bugzilla.AddComponent <channel> <component>");
		return;
	}
	var channel = String(params.get(1)).trim();
	
	var target = this._targetList[channel];
	if (!target) {
		target = this._targetList[channel] = new BugmailTarget(this, channel);
		this._mods.odb.save(target);
		target.__mods = this._mods;
		target.save = function _target_save() { this.__mods.odb.update(this) };
	}
	
	var component = String(params.get(2)).trim();
	
	if (target.addComponent(component)) {
		irc.sendContextReply(mes, "Channel '" + channel + "' will now receive bugmail for '" + component + "'.");
	} else {
		irc.sendContextReply(mes, "Channel '" + channel + "' already receives bugmail for '" + component + "'.");
	}
}
Bugzilla.prototype.commandAddComponent.help = [
		"Adds a new component to the list for a channel.",
		"<channel> <component>",
		"<channel> is the name of the channel to adjust",
		"<component> is the component to start sending bugmail for"
	];


// Command: RemoveComponent
Bugzilla.prototype.commandRemoveComponent = function(mes, mods, irc) {
	var params = mods.util.getParams(mes, 2);
	if (params.size() <= 2) {
		irc.sendContextReply(mes, "Syntax: Bugzilla.RemoveComponent <channel> <component>");
		return;
	}
	var channel = String(params.get(1)).trim();
	
	var target = this._targetList[channel];
	if (!target) {
		target = this._targetList[channel] = new BugmailTarget(this, channel);
		this._mods.odb.save(target);
		target.__mods = this._mods;
		target.save = function _target_save() { this.__mods.odb.update(this) };
	}
	
	var component = String(params.get(2)).trim();
	
	if (target.removeComponent(component)) {
		irc.sendContextReply(mes, "Channel '" + channel + "' will no longer receive bugmail for '" + component + "'.");
	} else {
		irc.sendContextReply(mes, "Channel '" + channel + "' doesn't receive bugmail for '" + component + "'.");
	}
}
Bugzilla.prototype.commandRemoveComponent.help = [
		"Removes a component from the list for a channel.",
		"<channel> <component>",
		"<channel> is the name of the channel to adjust",
		"<component> is the component to stop sending bugmail for"
	];


// Command: Log
Bugzilla.prototype.commandLog = function(mes, mods, irc) {
	var params = mods.util.getParams(mes, 1);
	if (params.size() <= 1) {
		irc.sendContextReply(mes, "Syntax: Bugzilla.Log <bug number>");
		return;
	}
	var bugNum = Number(params.get(1));
	
	var bugs = new Array();
	try {
		var self = this;
		this._checkMail(function(pop3, messageID) {
			var bug = new BugmailParser(pop3.getMessage(messageID));
			if ((bug.bugNumber == bugNum) || (bugNum == 0)) {
				bugs.push(bug);
			}
			return true;
		});
	} catch(ex) {
		irc.sendContextReply(mes, "Error checking bugmail: " + ex);
		return;
	}
	
	if (bugs.length == 0) {
		irc.sendContextReply(mes, "Nothing found in log for bug " + bugNum);
	} else {
		this._spam(bugs, mes);
	}
}
Bugzilla.prototype.commandLog.help = [
		"Shows all changes capture for a single bug.",
		"<bug number>",
		"<bug number> is the bug number to show the log of"
	];


// Command: Queue
Bugzilla.prototype.commandQueue = function(mes, mods, irc) {
	var params = mods.util.getParams(mes, 1);
	if (params.size() <= 1) {
		irc.sendContextReply(mes, "Syntax: Bugzilla.Queue <email>");
		return;
	}
	var email = String(params.get(1)).trim();
	
	var flags = this._mods.odb.retrieve(BugzillaSavedFlagRequest, "WHERE `from` = \"" + this._mods.odb.escapeString(email) + "\"");
	var list = new Array();
	
	for (var i = 0; i < flags.size(); i++) {
		var f = flags.get(i);
		list.push("bug " + f.bug + " (" + f.name + ")");
	}
	
	if (list.length == 0) {
		irc.sendContextReply(mes, "Queue is empty.");
	} else {
		irc.sendContextReply(mes, "Queue: " + list.join(", ") + ".");
	}
}
Bugzilla.prototype.commandQueue.help = [
		"Shows the review queue for a user, as seen by the bot.",
		"<email>",
		"<email> is the user to show the queue of"
	];


// Command: RebuildDB
Bugzilla.prototype.commandRebuildDB = function(mes, mods, irc) {
	var params = mods.util.getParams(mes, 0);
	if (params.size() <= 0) {
		irc.sendContextReply(mes, "Syntax: Bugzilla.RebuildDB");
		return;
	}
	
	// Delete all saved flags.
	var flags = this._mods.odb.retrieve(BugzillaSavedFlagRequest, "");
	for (var i = 0; i < flags.size(); i++)
		this._mods.odb["delete"](flags.get(i));
	
	var bugs = new Array();
	try {
		var self = this;
		this._checkMail(function(pop3, messageID) {
			var bug = new BugmailParser(pop3.getMessage(messageID));
			bugs.push(bug);
			return true;
		});
	} catch(ex) {
		irc.sendContextReply(mes, "Error checking bugmail: " + ex);
		return;
	}
	
	this._updateActivityDB(bugs);
	this._updateFlagDB(bugs);
	irc.sendContextReply(mes, "Database rebuilt from bugmail.");
}
Bugzilla.prototype.commandRebuildDB.help = [
		"Rebuilds the list of requested flags.",
		""
	];


// Internal: calls back for each message.
Bugzilla.prototype._checkMail = function(callbackFn) {
	var pop3host     = this._mods.plugin.callAPI("Options", "GetGeneralOption", ["POP3Server",   ""]);
	var pop3port     = this._mods.plugin.callAPI("Options", "GetGeneralOption", ["POP3Port",     "110"]);
	var pop3account  = this._mods.plugin.callAPI("Options", "GetGeneralOption", ["POP3Account",  ""]);
	var pop3password = this._mods.plugin.callAPI("Options", "GetGeneralOption", ["POP3Password", ""]);
	
	var pop3 = new POP3Server(pop3host, pop3port, pop3account, pop3password);
	var messageIDList = pop3.getMessageList();
	for (var i = 0; i < messageIDList.length; i++) {
		if (!callbackFn(pop3, messageIDList[i])) {
			break;
		}
	}
	pop3.close();
}


// Internal: spam list of bugs at their appropriate channels.
Bugzilla.prototype._spam = function(bugs, mes) {
	for (var i = 0; i < bugs.length; i++) {
		var things = new Array();
		
		if (bugs[i].isNew) {
			things.push("filed by " + bugs[i].from);
		}
		
		if (bugs[i].newAttachment) {
			var msgp = "added attachment " + bugs[i].newAttachment.number;
			if (bugs[i].newAttachment.label) {
				msgp += " (" + bugs[i].newAttachment.label + ")";
			}
			things.push(msgp);
		}
		
		if (bugs[i].changes.length > 0) {
			var nameMapping = new Object();
			var ignoredChanges = new Object();
			var resolved = "";
			var reopened = "";
			
			for (var j = 0; j < bugs[i].changes.length; j++) {
				if (bugs[i].changes[j].name == "Status") {
					nameMapping["status"] = j;
				}
				if (bugs[i].changes[j].name == "Resolution") {
					nameMapping["resolution"] = j;
				}
			}
			
			if (("status" in nameMapping) && ("resolution" in nameMapping)) {
				var s = bugs[i].changes[nameMapping["status"]];
				var r = bugs[i].changes[nameMapping["resolution"]];
				
				if ((s.newValue == "RESOLVED") && r.newValue) {
					// RESOLVED the bug, so ignore status/resolution changes.
					resolved = r.newValue;
					ignoredChanges[nameMapping["status"]] = true;
					ignoredChanges[nameMapping["resolution"]] = true;
					
				} else if ((s.oldValue == "RESOLVED") && (s.newValue == "REOPENED") && r.oldValue) {
					// REOPENED the bug, so ignore status/resolution changes.
					reopened = true;
					ignoredChanges[nameMapping["status"]] = true;
					ignoredChanges[nameMapping["resolution"]] = true;
				}
			}
			
			if (resolved) {
				things.push("resolved " + resolved);
			} else if (reopened) {
				things.push("reopened");
			}
			
			var list = new Array();
			for (var j = 0; j < bugs[i].changes.length; j++) {
				if ((j in ignoredChanges) && ignoredChanges[j]) {
					continue;
				}
				list.push(bugs[i].changes[j].name
						+ (bugs[i].changes[j].oldValue ? " from '" + bugs[i].changes[j].oldValue + "'" : "")
						+ " to '" + bugs[i].changes[j].newValue + "'");
			}
			if (list.length > 0) {
				things.push("changed " + list.join(", "));
			}
		}
		
		if (bugs[i].removed.length > 0) {
			var list = new Array();
			for (var j = 0; j < bugs[i].removed.length; j++) {
				list.push(bugs[i].removed[j].name + " '" + bugs[i].removed[j].value + "'");
			}
{'fraction_non_alphanumeric': 0.17558590145783354, 'fraction_numerical': 0.00867318693485883, 'mean_word_length': 3.6072263549415515, 'pattern_counts': {'":': 12, '<': 75, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '765368', 'n_tokens_mistral': 8558, 'n_tokens_neox': 7507, 'n_words': 1695}