<commit_before>"use strict";

var base = require("xbase"),
	fs = require("fs"),
	path = require("path"),
	util = require("util"),
	uuid = require("node-uuid"),
	tiptoe = require("tiptoe");

exports.searchReplace = searchReplace;
function searchReplace(file, match, replace, cb)
{
	tiptoe(
		function loadFile()
		{
			fs.readFile(file, "utf8", this);
		},
		function replaceAndSave(data)
		{
			fs.writeFile(file, data.replace(new RegExp(match, "g"), replace), "utf8", this);
		},
		function handleErrors(err) { setImmediate(function() { cb(err); }); }
	);
}

exports.concat = concat;
function concat(files, dest, options, cb)
{
	if(!cb && typeof(options)==="function")
	{
		cb = options;
		options = {};
	}

	var writeSeperator = 1;

	files = files.slice();

	base.info("Combining to [%s] files: %s", dest, files.join(" "));

	var output = fs.createWriteStream(dest);

	var first = true;
	function concatNext()
	{
		if(first && options.prefix)
			output.write(options.prefix);

		if(options.filePrefixes && files.length)
			output.write(options.filePrefixes[writeSeperator-1]);

		first = false;

		if(options.seperator && (writeSeperator%2)===0)
			output.write(options.seperator);

		writeSeperator++;

		if(!files.length)
		{
			if(options.suffix)
				output.write(options.suffix);

			output.end();
			return cb();
		}

		var input = fs.createReadStream(files.shift());
		input.pipe(output, { end : false });
		input.on("end", concatNext);
	}

	concatNext();
}

exports.copy = copy;
function copy(src, dest, cb)
{
	var cbCalled = false;

	var rd = fs.createReadStream(src);
	rd.on("error", function(err) { done(err); });

	var wr = fs.createWriteStream(dest);
	wr.on("error", function(err) { done(err); });

	wr.on("close", function(ex) { done(); });
	rd.pipe(wr);

	function done(err)
	{
		if(!cbCalled)
		{
			cb(err);
			cbCalled = true;
		}
	}
}

exports.move = move;
function move(src, dest, cb)
{
	tiptoe(
		function checkExisting()
		{
			exports.exists(dest, this);
		},
		function removeExisting(exists)
		{
			if(exists)
				fs.unlink(dest, this);
			else
				this();
		},
		function copyFile()
		{
			copy(src, dest, this);
		},
		function removeFile()
		{
			fs.unlink(src, this);
		},
		function handleErrors(err) { setImmediate(function() { cb(err); }); }
	);
}

exports.generateTempFilePath = generateTempFilePath;
function generateTempFilePath(prefix)
{
	var tempFilePath;
	var existsSync = fs.existsSync ? fs.existsSync : path.existsSync;
	prefix = prefix || "/tmp";

	do
	{
		tempFilePath = path.join(prefix, uuid.v4() + ".tmp");
	} while(existsSync(tempFilePath));

	return tempFilePath;
}

exports.exists = exists;
function exists(target, cb)
{
	fs.exists(target, function(exists) { cb(null, exists); });
}

exports.unlink = unlink;
function unlink(target, cb)
{
	fs.exists(target, function(exists) { if(exists) { fs.unlink(target, cb); } else { return setImmediate(cb); }});
}<commit_msg>Added suffix support to tmp file<commit_after>"use strict";

var base = require("xbase"),
	fs = require("fs"),
	path = require("path"),
	util = require("util"),
	uuid = require("node-uuid"),
	tiptoe = require("tiptoe");

exports.searchReplace = searchReplace;
function searchReplace(file, match, replace, cb)
{
	tiptoe(
		function loadFile()
		{
			fs.readFile(file, {encoding:"utf8"}, this);
		},
		function replaceAndSave(data)
		{
			fs.writeFile(file, data.replace(new RegExp(match, "g"), replace), {encoding:"utf8"}, this);
		},
		function handleErrors(err) { setImmediate(function() { cb(err); }); }
	);
}

exports.concat = concat;
function concat(files, dest, options, cb)
{
	if(!cb && typeof(options)==="function")
	{
		cb = options;
		options = {};
	}

	var writeSeperator = 1;

	files = files.slice();

	base.info("Combining to [%s] files: %s", dest, files.join(" "));

	var output = fs.createWriteStream(dest);

	var first = true;
	function concatNext()
	{
		if(first && options.prefix)
			output.write(options.prefix);

		if(options.filePrefixes && files.length)
			output.write(options.filePrefixes[writeSeperator-1]);

		first = false;

		if(options.seperator && (writeSeperator%2)===0)
			output.write(options.seperator);

		writeSeperator++;

		if(!files.length)
		{
			if(options.suffix)
				output.write(options.suffix);

			output.end();
			return cb();
		}

		var input = fs.createReadStream(files.shift());
		input.pipe(output, { end : false });
		input.on("end", concatNext);
	}

	concatNext();
}

exports.copy = copy;
function copy(src, dest, cb)
{
	var cbCalled = false;

	var rd = fs.createReadStream(src);
	rd.on("error", function(err) { done(err); });

	var wr = fs.createWriteStream(dest);
	wr.on("error", function(err) { done(err); });

	wr.on("close", function(ex) { done(); });
	rd.pipe(wr);

	function done(err)
	{
		if(!cbCalled)
		{
			cb(err);
			cbCalled = true;
		}
	}
}

exports.move = move;
function move(src, dest, cb)
{
	tiptoe(
		function checkExisting()
		{
			exports.exists(dest, this);
		},
		function removeExisting(exists)
		{
			if(exists)
				fs.unlink(dest, this);
			else
				this();
		},
		function copyFile()
		{
			copy(src, dest, this);
		},
		function removeFile()
		{
			fs.unlink(src, this);
		},
		function handleErrors(err) { setImmediate(function() { cb(err); }); }
	);
}

exports.generateTempFilePath = generateTempFilePath;
function generateTempFilePath(prefix, suffix)
{
	var tempFilePath;
	var existsSync = fs.existsSync ? fs.existsSync : path.existsSync;
	prefix = prefix || "/tmp";

	do
	{
		tempFilePath = path.join(prefix, uuid.v4() + (typeof suffix==="undefined" ? ".tmp" : suffix));
	} while(existsSync(tempFilePath));

	return tempFilePath;
}

exports.exists = exists;
function exists(target, cb)
{
	fs.exists(target, function(exists) { cb(null, exists); });
}

exports.unlink = unlink;
function unlink(target, cb)
{
	fs.exists(target, function(exists) { if(exists) { fs.unlink(target, cb); } else { return setImmediate(cb); }});
}
{'fraction_non_alphanumeric': 0.175397093612707, 'fraction_numerical': 0.0023656640757012504, 'mean_word_length': 4.238053097345133, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7398495', 'n_tokens_mistral': 2297, 'n_tokens_neox': 2105, 'n_words': 421}