<commit_before>var fs = require('fs');
var path = require('path');
var mkdirp = require('mkdirp');
var rimraf = require('rimraf');
var thunky = require('thunky');
var events = require('events');

var noop = function() {};

module.exports = function(folder, torrent) {
	var that = new events.EventEmitter();

	var mkdir = thunky(function(cb) {
		mkdirp(folder, cb);
	});

	var pad = function(i) {
		return '00000000000'.slice(0, 10-(''+i).length)+i;
	};

	that.read = function(index, cb) {
		mkdir(function(err) {
			if (err) return cb(err);
			fs.readFile(path.join(folder, pad(index)), cb);
		});
	};

	that.write = function(index, buffer, cb) {
		if (!cb) cb = noop;
		mkdir(function(err) {
			if (err) return cb(err);
			fs.writeFile(path.join(folder, pad(index)), buffer, cb);
		});
	};

	that.destroy = function(cb) {
		if (!cb) cb = noop;
		mkdir(function(err) {
			if (err) return cb(err);
			rimraf(folder, cb);<commit_msg>using 256mb buffers for less fd open/close<commit_after>var fs = require('fs');
var path = require('path');
var mkdirp = require('mkdirp');
var rimraf = require('rimraf');
var thunky = require('thunky');
var events = require('events');
var raf = require('random-access-file');

var noop = function() {};

module.exports = function(folder, torrent) {
	var that = new events.EventEmitter();

	var bufferSize = torrent.pieceLength;
	while (bufferSize < 256 * 1024 * 1024) bufferSize *= 2;

	var pieceLength = torrent.pieceLength;
	var pieceRemainder = (torrent.length % pieceLength) || pieceLength;
	var piecesPerBuffer = bufferSize / pieceLength;

	var mkdir = thunky(function(cb) {
		mkdirp(folder, cb);
	});

	var pad = function(i) {
		return '00000000000'.slice(0, 10-(''+i).length)+i;
	};

	var files = [];

	that.read = function(index, cb) {
		mkdir(function(err) {
			if (err) return cb(err);

			var i = (index / piecesPerBuffer) | 0;
			var offset = index - i * piecesPerBuffer;
			var len = index === torrent.pieces.length-1 ? pieceRemainder : pieceLength;
			var file = files[i] = files[i] || raf(path.join(folder, pad(i)));

			file.read(offset * pieceLength, len, cb);
		});
	};

	that.write = function(index, buffer, cb) {
		if (!cb) cb = noop;
		mkdir(function(err) {
			if (err) return cb(err);

			var i = (index / piecesPerBuffer) | 0;
			var file = files[i] = files[i] || raf(path.join(folder, pad(i)));
			var offset = index - i * piecesPerBuffer;

			file.write(offset * pieceLength, buffer, cb);
		});
	};

	that.destroy = function(cb) {
		if (!cb) cb = noop;

		files.forEach(function(file) {
			file.close();
		});

		mkdir(function(err) {
			if (err) return cb(err);
			rimraf(folder, cb);
{'fraction_non_alphanumeric': 0.17676003028009085, 'fraction_numerical': 0.0174110522331567, 'mean_word_length': 4.024714828897339, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '556100', 'n_tokens_mistral': 1062, 'n_tokens_neox': 929, 'n_words': 233}