<commit_before>var fs = require("fs"),
	uuid = require("uuid"),
	request = require("sync-request");

const directory = process.argv[3];

var urls = [],
	snippets = {},
	resources = {},
	files = [];

var logSnippet = function(filename, logContent, content) {
	snippets[filename] = logContent;
	fs.writeFileSync(directory + filename, content);
	fs.writeFileSync(directory + "snippets.json", JSON.stringify(snippets, null, "\t"));
};

module.exports = {
	getUUID: uuid.v4,
	kill: function(message) {
		console.trace(message)
		process.exit(0)
	},
	fetchUrl: function(method, url, headers = {}, body) {
		if (process.argv.indexOf("--download") == -1) {
			console.log(`Faking a ${method} request to ${url}`);
			console.log("Use the flag --download to actually download the file (eg. for encoded payloads).");
			return `(Content of ${url})`;
		}

		console.log(`Emulating a ${method} request to ${url}`);
		headers["User-Agent"] = "Mozilla/4.0 (Windows; MSIE 6.0; Windows NT 6.0)";
		let options = {
			headers,
			maxRedirects: 20,
			timeout: 5000
		};
		if (body)
			options.body = body;
		var file = request(method, url, options);
		Buffer.prototype.charCodeAt = function(index) { return this[index]; }
		console.log("Length: " + file.body.length + " bytes.");
		return file.body;
	},
	writeFile: function(filename, contents) {
		files[filename] = contents;
	},
	readFile: function(filename) {
		return files[filename];
	},
	logUrl: function(method, url) {
		if (urls.indexOf(url) == -1) urls.push(url);
		fs.writeFileSync(directory + "urls.json", JSON.stringify(urls, null, "\t"))
	},
	logResource: function(resourceName, logContent, content) {
		resources[resourceName] = logContent;
		fs.writeFileSync(directory + resourceName, content);
		fs.writeFileSync(directory + "resources.json", JSON.stringify(resources, null, "\t"))
	},
	logSnippet,
	logJS: function(code) {
		const filename = uuid.v4() + ".js";
		console.log("Code saved to", filename)
		logSnippet(filename, {as: "eval'd JS"}, code)
		return code; // Helps with tail call optimization
	}
}<commit_msg>Print resource filetype when saving<commit_after>var fs = require("fs"),
	uuid = require("uuid"),
	request = require("sync-request");

const directory = process.argv[3];

var urls = [],
	snippets = {},
	resources = {},
	files = [];

var logSnippet = function(filename, logContent, content) {
	snippets[filename] = logContent;
	fs.writeFileSync(directory + filename, content);
	fs.writeFileSync(directory + "snippets.json", JSON.stringify(snippets, null, "\t"));
};

module.exports = {
	getUUID: uuid.v4,
	kill: function(message) {
		console.trace(message)
		process.exit(0)
	},
	fetchUrl: function(method, url, headers = {}, body) {
		if (process.argv.indexOf("--download") == -1) {
			console.log(`Faking a ${method} request to ${url}`);
			console.log("Use the flag --download to actually download the file (eg. for encoded payloads).");
			return `(Content of ${url})`;
		}

		console.log(`Emulating a ${method} request to ${url}`);
		headers["User-Agent"] = "Mozilla/4.0 (Windows; MSIE 6.0; Windows NT 6.0)";
		let options = {
			headers,
			maxRedirects: 20,
			timeout: 5000
		};
		if (body)
			options.body = body;
		var file = request(method, url, options);
		Buffer.prototype.charCodeAt = function(index) { return this[index]; }
		console.log(`Downloaded ${file.body.length} bytes.`);
		return file.body;
	},
	writeFile: function(filename, contents) {
		files[filename] = contents;
	},
	readFile: function(filename) {
		return files[filename];
	},
	logUrl: function(method, url) {
		if (urls.indexOf(url) == -1) urls.push(url);
		fs.writeFileSync(directory + "urls.json", JSON.stringify(urls, null, "\t"))
	},
	logResource: function(resourceName, logContent, content, print = false) {
		resources[resourceName] = logContent;
		fs.writeFileSync(directory + resourceName, content);
		fs.writeFileSync(directory + "resources.json", JSON.stringify(resources, null, "\t"))
		if (!print) return;
		let filetype = require("child_process").execSync("file " + JSON.stringify(directory + resourceName)).toString("utf8");
		filetype = filetype.replace(`${directory + resourceName}: `, "").replace("\n", "");
		console.log(`Saved ${directory + resourceName} (${content.length})`);
		console.log(`${directory + resourceName} has been detected as ${filetype}.`);
	},
	logSnippet,
	logJS: function(code) {
		const filename = uuid.v4() + ".js";
		console.log("Code saved to", filename)
		logSnippet(filename, {as: "eval'd JS"}, code)
		return code; // Helps with tail call optimization
	}
}
{'fraction_non_alphanumeric': 0.17252747252747253, 'fraction_numerical': 0.008131868131868131, 'mean_word_length': 4.611590628853268, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2703293', 'n_tokens_mistral': 1597, 'n_tokens_neox': 1506, 'n_words': 359}