<commit_before>var fs = require('fs');
var path = require('path');

var tmp = require('tmp');
var request = require('request');

exports.readThenDeleteLocalFile = function (filepath, callback) {
	fs.readFile(filepath, function (err, data) {
		fs.unlink(filepath, function (err) {
			if (!err) {
				console.log("successfully deleted " + filepath);
			}
		});
		callback(err, data ? data : null);
	});
};

exports.getSize = function (filepath, callback) {
	fs.stat(filepath, function (err, stats) {
		callback(err, stats ? stats.size : 0);
	});
};

exports.writeDataToFileSystem = function (filename, data, callback) {
	tmp.dir(function (err, directoryPath) {
		if (err) {
			console.log("err: ");
			console.log(err);
			return callback(err);
		}

		var filepath = directoryPath + "/" + filename;
		fs.writeFile(filepath, data, function (err) {
			callback(err, filepath);
		});
	});
};

exports.createStreamToFileSystem = function (callback) {
	tmp.file(function (err, filepath) {
		if (err) {
			return callback(err);
		}

		callback(null, filepath, fs.createWriteStream(filepath));
	});
};

exports.getFileFromUrl = function (url, callback) {
	exports.createStreamToFileSystem(function (err, filepath, stream) {
		if (err) {
			return callback(err);
		}

		request(url, function (err, response, body) {
			if (err) {
				return callback(err);
			}

			var filename = _getFilename(response);

			console.log("filename:", filename);

			stream.on('finish', function () {
				console.log('file downloaded');

				callback(null, filepath, filename);
			});
		}).pipe(stream);
	});
};

function _getFilename(response) {
	console.log("headers:", response.headers);

	// TODO sanitize inputs

	if (response.headers['content-type'] && !! ~response.headers['content-type'].indexOf('name')) {
		return _extractFilenameFromHeaders(response.headers['content-type'], 'name');
	} else if (response.headers['content-disposition'] && !! ~response.headers['content-disposition'].indexOf('filename')) {
		return _extractFilenameFromHeaders(response.headers['content-disposition'], 'filename');
	} else {
		return response.req.path.indexOf('/') === 0 ? response.req.path.slice(1) : response.req.path;
	}
}

function _extractFilenameFromHeaders(header, key) {
	var item = header.slice(header.indexOf(key));
	return item.slice(item.indexOf('"') + 1, -1) || item.slice(item.indexOf('\'') + 1, -1);
}<commit_msg>fix get filename error<commit_after>var fs = require('fs');
var path = require('path');

var tmp = require('tmp');
var request = require('request');

exports.readThenDeleteLocalFile = function (filepath, callback) {
	fs.readFile(filepath, function (err, data) {
		fs.unlink(filepath, function (err) {
			if (!err) {
				console.log("successfully deleted " + filepath);
			}
		});
		callback(err, data ? data : null);
	});
};

exports.getSize = function (filepath, callback) {
	fs.stat(filepath, function (err, stats) {
		callback(err, stats ? stats.size : 0);
	});
};

exports.writeDataToFileSystem = function (filename, data, callback) {
	tmp.dir(function (err, directoryPath) {
		if (err) {
			console.log("err: ");
			console.log(err);
			return callback(err);
		}

		var filepath = directoryPath + "/" + filename;
		fs.writeFile(filepath, data, function (err) {
			callback(err, filepath);
		});
	});
};

exports.createStreamToFileSystem = function (callback) {
	tmp.file(function (err, filepath) {
		if (err) {
			return callback(err);
		}

		callback(null, filepath, fs.createWriteStream(filepath));
	});
};

exports.getFileFromUrl = function (url, callback) {
	exports.createStreamToFileSystem(function (err, filepath, stream) {
		if (err) {
			return callback(err);
		}

		request(url, function (err, response, body) {
			if (err) {
				return callback(err);
			}

			var filename;
			try {
				filename = _getFilename(response);
			} catch (e) {
				return callback(e);
			}

			console.log("filename:", filename);

			stream.on('finish', function () {
				console.log('file downloaded');

				callback(null, filepath, filename);
			});
		}).pipe(stream);
	});
};

function _getFilename(response) {
	console.log("headers:", response.headers);

	// TODO sanitize inputs

	if (response.headers['content-type'] && !!~response.headers['content-type'].indexOf('name')) {
		return _extractFilenameFromHeaders(response.headers['content-type'], 'name');
	} else if (response.headers['content-disposition'] && !!~response.headers['content-disposition'].indexOf('filename')) {
		return _extractFilenameFromHeaders(response.headers['content-disposition'], 'filename');
	} else {
		////////////////////////////////////////////////////////////////
		// TODO https://app.asana.com/0/11244390559721/14733882770247 //
		////////////////////////////////////////////////////////////////

		throw new SGError('NO_FILENAME_FOUND');
	}
}

function _extractFilenameFromHeaders(header, key) {
	var item = header.slice(header.indexOf(key));
	return item.slice(item.indexOf('"') + 1, -1) || item.slice(item.indexOf('\'') + 1, -1);
}
{'fraction_non_alphanumeric': 0.19408945686900958, 'fraction_numerical': 0.008186900958466454, 'mean_word_length': 3.9447186574531097, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4411378', 'n_tokens_mistral': 1745, 'n_tokens_neox': 1564, 'n_words': 317}