<commit_before>const tar = require('tar-stream')
const path = require('path')
const file = process.argv[2] || path.resolve(__dirname, '../npm.tar')
const fs = require('fs')

let slices = 0
Buffer.prototype.slice = (original => function () {
  slices ++
  return original.apply(this, arguments)
})(Buffer.prototype.slice)

const start = process.hrtime()
const p = tar.extract()
p.on('entry', (entry, stream, callback) => {
  stream.on('end', callback)
  stream.resume()
})
p.on('data', _ => _)
process.on('exit', (code, signal) => {
  const end = process.hrtime(start)
  console.log(end[0]*1e3 + end[1]/1e6)
  console.log(slices)
})
fs.createReadStream(file).pipe(p)<commit_msg>benchmark scripts<commit_after>const tar = require('tar-stream')
const path = require('path')
const file = process.argv[2] || path.resolve(__dirname, '../npm.tar')
const fs = require('fs')

const start = process.hrtime()
const p = tar.extract()
p.on('entry', (entry, stream, callback) => {
  stream.on('end', callback)
  stream.resume()
})
p.on('data', _ => _)
process.on('exit', (code, signal) => {
  const end = process.hrtime(start)
  console.log(end[0]*1e3 + end[1]/1e6)
})
fs.createReadStream(file).pipe(p)
{'fraction_non_alphanumeric': 0.20117845117845118, 'fraction_numerical': 0.012626262626262626, 'mean_word_length': 5.035532994923858, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1530131', 'n_tokens_mistral': 423, 'n_tokens_neox': 423, 'n_words': 86}