<commit_before>
var sandbox = require("sandboxed-module");

exports["util.tokenize"] = function(test){
	
	var util = require("../lib/util");
	var str = ["this is a test1,test2;test3,test4 test5"];
	var tokens = util.tokenize(str);
	test.deepEqual(tokens,["this","is","a","test1","test2","test3","test4","test5"]);
	test.deepEqual(util.tokenize("foo;bar dummy"),["foo","bar","dummy"]);
	test.done();
	
}

exports["util.has_joined: true"] = function(test){
	
	var util = require("../lib/util");
	var rcpts_arr = [{push_id:"123", push_type:"web"},{push_id:"456",push_type:"gcm"},{push_id:"789",push_type:"gcm"}];
	test.ok(util.has_joined(rcpts_arr,"123"));
	test.done();
	
}

exports["util.has_joined: false"] = function(test){
	
	var util = require("../lib/util");
	var rcpts_arr = [{push_id:"123", push_type:"web"},{push_id:"456",push_type:"gcm"},{push_id:"789",push_type:"gcm"}];
	test.equal( false, util.has_joined(rcpts_arr,"124"));
	test.done();
	
}


exports["util.has_joined: undefined"] = function(test){
	
	var util = require("../lib/util");
	var rcpts_arr = [{push_id:"123", push_type:"web"},{push_id:"456",push_type:"gcm"},{push_id:"789",push_type:"gcm"}];<commit_msg>extended util.tokenize test.<commit_after>
var sandbox = require("sandboxed-module");

exports["util.tokenize"] = function(test){
	
	var util = require("../lib/util");
	var str = ["this is a test1,test2;test3,test4 test5.test6 test7"];
	var tokens = util.tokenize(str);
	test.deepEqual(tokens,["this","is","a","test1","test2","test3","test4","test5","test6","test7"]);
	test.deepEqual(util.tokenize("foo;bar dummy"),["foo","bar","dummy"]);
	test.done();
	
}

exports["util.has_joined: true"] = function(test){
	
	var util = require("../lib/util");
	var rcpts_arr = [{push_id:"123", push_type:"web"},{push_id:"456",push_type:"gcm"},{push_id:"789",push_type:"gcm"}];
	test.ok(util.has_joined(rcpts_arr,"123"));
	test.done();
	
}

exports["util.has_joined: false"] = function(test){
	
	var util = require("../lib/util");
	var rcpts_arr = [{push_id:"123", push_type:"web"},{push_id:"456",push_type:"gcm"},{push_id:"789",push_type:"gcm"}];
	test.equal( false, util.has_joined(rcpts_arr,"124"));
	test.done();
	
}


exports["util.has_joined: undefined"] = function(test){
	
	var util = require("../lib/util");
	var rcpts_arr = [{push_id:"123", push_type:"web"},{push_id:"456",push_type:"gcm"},{push_id:"789",push_type:"gcm"}];
{'fraction_non_alphanumeric': 0.24488944513975802, 'fraction_numerical': 0.03754693366708386, 'mean_word_length': 7.240549828178694, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5958597', 'n_tokens_mistral': 984, 'n_tokens_neox': 918, 'n_words': 120}