<commit_before>var assert = require('assert')

var Atok = require('atok')
var newAtok = require('..')

var atok = new Atok
var natok = new newAtok

function handler (token, idx, type) {
	// assert.equal(token, 'abc')
}

atok.addRule(3, handler)
natok.addRule(3, handler)

// Converting a string to a buffer is *expensive*
// a lot more than vice-versa
var s = 'abc'
var buf = new Buffer(s)

var compare = exports.compare = {}
compare[Atok.version] = function () {
	atok.clear(true).write(s)
}
compare[newAtok.version + ' string'] = function () {
	natok.clear(true).write(s)
}
compare[newAtok.version + ' buffer'] = function () {
	natok.clear(true).write(buf)
}
require("bench").runMain()<commit_msg>Removed .clear calls<commit_after>var assert = require('assert')

var Atok = require('atok')
var newAtok = require('..')

var atok = new Atok
var natok = new newAtok

function handler (token, idx, type) {
	// assert.equal(token, 'abc')
}

atok.addRule(3, handler)
natok.addRule(3, handler)

// Converting a string to a buffer is *expensive*
// a lot more than vice-versa
var s = 'abc'
var buf = new Buffer(s)

var compare = exports.compare = {}
compare[Atok.version] = function () {
	atok.write(s)
}
compare[newAtok.version + ' string'] = function () {
	natok.write(s)
}
compare[newAtok.version + ' buffer'] = function () {
	natok.write(buf)
}
require("bench").runMain()
{'fraction_non_alphanumeric': 0.16508400292184075, 'fraction_numerical': 0.0029218407596785976, 'mean_word_length': 4.41501976284585, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3059786', 'n_tokens_mistral': 497, 'n_tokens_neox': 492, 'n_words': 127}