<commit_before>#!/usr/bin/env node

var chokidar = require("chokidar");
var fs = require("fs");
var http = require("http");
var url = require("url");
var path = require("path");
var util = require("util");

var SOURCE_DIR;

var pkgjson = require("./package.json");
var program = require("commander");
program
	.version(pkgjson.version)
	.usage("rbxrefresh [options] <dir>")
	.arguments("<dir>")
	.option("-s, --sync", "Enables sync to filesystem")
	.action(function(env) {
		SOURCE_DIR = env;
	})
	.parse(process.argv);

var SRC_UTILITY_FUNC_LUA = fs.readFileSync(
	path.resolve(
		__dirname,
		"templates/UtilityFuncLua.template.lua")).toString();

var SRC_SET_SOURCE_CALL_LUA = fs.readFileSync(
	path.resolve(
		__dirname,
		"templates/SetSourceCall.template.lua")).toString();

var SRC_REMOVE_FILE_CALL_LUA = fs.readFileSync(
	path.resolve(
		__dirname,
		"templates/RemoveFileCall.template.lua")).toString()

var SRC_PRINT_LUA = fs.readFileSync(
	path.resolve(
		__dirname,
		"templates/Print.template.lua")).toString()

var SRC_SYNC_TO_FS_LUA = fs.readFileSync(
	path.resolve(
		__dirname,
		"templates/SyncToFs.template.lua")).toString()

var FSEXT_LUA = ".lua";

var RBXTYPE_MODULESCRIPT = "ModuleScript"
var RBXTYPE_LOCALSCRIPT = "LocalScript"
var RBXTYPE_SCRIPT = "Script"

function generateUpdateAllFilesCode_rTraversal(dir, outCodeLines) {
	fs.readdirSync(dir).forEach(function(itrFileName) {
		var itrFilePath = path.resolve(dir, itrFileName);
		if (fs.statSync(itrFilePath).isDirectory()) {
			generateUpdateAllFilesCode_rTraversal(itrFilePath, outCodeLines);
		} else {
			var fileExt = path.extname(itrFilePath);
			if (fileExt == FSEXT_LUA) {
				outCodeLines.push(generateUpdateFileCode(itrFilePath));
			}
		}
	});
}

function generateUpdateAllFilesCodeLines(dir) {
	var outCodeLines = [];
	generateUpdateAllFilesCode_rTraversal(dir, outCodeLines);
	return outCodeLines;
}

function jsArrayToLuaArrayString(jsarray) {
	return "{" + jsarray.map(function(x) { return "\"" + x + "\"" }).join() + "}";
}

function matchAssetRbxType(str) {
	if (str == RBXTYPE_LOCALSCRIPT) { return RBXTYPE_LOCALSCRIPT; }
	if (str == RBXTYPE_SCRIPT) { return RBXTYPE_SCRIPT; }
	if (str == RBXTYPE_MODULESCRIPT) { return RBXTYPE_MODULESCRIPT; }
	console.warn("Unknown file subext:" + str);
	return RBXTYPE_MODULESCRIPT;
}

function getAssetRbxInfoFromFilepath(filepath) {
	var assetFullName = path.basename(filepath, FSEXT_LUA);
	var assetRbxName = "";
	var assetRbxType = path.extname(assetFullName).replace(".", "");

	if (assetRbxType == "") {
		if (filepath.indexOf("ServerScriptService") != -1) {
			assetRbxType = RBXTYPE_SCRIPT;
		} else if (filepath.indexOf("StarterPlayer") != -1) {
			assetRbxType = RBXTYPE_LOCALSCRIPT;
		} else {
			assetRbxType = RBXTYPE_MODULESCRIPT;
		}
		assetRbxName = assetFullName;

	} else {
		assetRbxName = path.basename(assetFullName, "." + assetRbxType);
		assetRbxType = matchAssetRbxType(assetRbxType);
	}

	var relativeFilepathArray = path.relative(SOURCE_DIR, filepath).split(path.sep);
	relativeFilepathArray.pop();

	return {
		RbxName: assetRbxName,
		RbxType: assetRbxType,<commit_msg>Added support for subext aliases.<commit_after>#!/usr/bin/env node

var chokidar = require("chokidar");
var fs = require("fs");
var http = require("http");
var url = require("url");
var path = require("path");
var util = require("util");

var SOURCE_DIR;

var pkgjson = require("./package.json");
var program = require("commander");
program
	.version(pkgjson.version)
	.usage("rbxrefresh [options] <dir>")
	.arguments("<dir>")
	.option("-s, --sync", "Enables sync to filesystem")
	.action(function(env) {
		SOURCE_DIR = env;
	})
	.parse(process.argv);

function getTemplate(templatePath) {
	return fs.readFileSync(path.resolve(__dirname, templatePath));
}

var SRC_UTILITY_FUNC_LUA = getTemplate("templates/UtilityFuncLua.template.lua");
var SRC_SET_SOURCE_CALL_LUA = getTemplate("templates/SetSourceCall.template.lua");
var SRC_REMOVE_FILE_CALL_LUA = getTemplate("templates/RemoveFileCall.template.lua");
var SRC_PRINT_LUA = getTemplate("templates/Print.template.lua");
var SRC_SYNC_TO_FS_LUA = getTemplate("templates/SyncToFs.template.lua");

var FSEXT_LUA = ".lua";

var RBXTYPE_MODULESCRIPT_ALIASES = ["ModuleScript", "module"];
var RBXTYPE_LOCALSCRIPT_ALIASES = ["LocalScript", "local", "client"];
var RBXTYPE_SCRIPT_ALIASES = ["Script", "server", ""];

var RBXTYPE_MODULESCRIPT = "ModuleScript"
var RBXTYPE_LOCALSCRIPT = "LocalScript"
var RBXTYPE_SCRIPT = "Script"

function isAliasOf(str, aliases) {
	for (var i = 0; i < aliases.length; i++) {
		if (aliases[i].toLowerCase() == str.toLowerCase()) {
			return true;
		}
	}
	return false;
}

function generateUpdateAllFilesCode_rTraversal(dir, outCodeLines) {
	fs.readdirSync(dir).forEach(function(itrFileName) {
		var itrFilePath = path.resolve(dir, itrFileName);
		if (fs.statSync(itrFilePath).isDirectory()) {
			generateUpdateAllFilesCode_rTraversal(itrFilePath, outCodeLines);
		} else {
			var fileExt = path.extname(itrFilePath);
			if (fileExt == FSEXT_LUA) {
				outCodeLines.push(generateUpdateFileCode(itrFilePath));
			}
		}
	});
}

function generateUpdateAllFilesCodeLines(dir) {
	var outCodeLines = [];
	generateUpdateAllFilesCode_rTraversal(dir, outCodeLines);
	return outCodeLines;
}

function jsArrayToLuaArrayString(jsarray) {
	return "{" + jsarray.map(function(x) { return "\"" + x + "\"" }).join() + "}";
}

function matchAssetRbxType(str) {
	if (isAliasOf(str, RBXTYPE_LOCALSCRIPT_ALIASES)) {
		return RBXTYPE_LOCALSCRIPT;
	} else if (isAliasOf(str, RBXTYPE_SCRIPT_ALIASES)) {
		return RBXTYPE_SCRIPT_ALIASES;
	} else if (isAliasOf(str, RBXTYPE_MODULESCRIPT_ALIASES)) {
		return RBXTYPE_MODULESCRIPT;
	} else {
		console.warn("Unknown file subext:" + str);
		return RBXTYPE_MODULESCRIPT;
	}
}

function getAssetRbxInfoFromFilepath(filepath) {
	var assetFullName = path.basename(filepath, FSEXT_LUA);
	var assetRbxName = "";
	var assetRbxType = path.extname(assetFullName).replace(".", "");

	if (assetRbxType == "") {
		if (filepath.indexOf("ServerScriptService") != -1) {
			assetRbxType = RBXTYPE_SCRIPT;
		} else if (filepath.indexOf("StarterPlayer") != -1) {
			assetRbxType = RBXTYPE_LOCALSCRIPT;
		} else {
			assetRbxType = RBXTYPE_MODULESCRIPT;
		}
		assetRbxName = assetFullName;

	} else {
		assetRbxName = path.basename(assetFullName, "." + assetRbxType);
		assetRbxType = matchAssetRbxType(assetRbxType);
	}

	var relativeFilepathArray = path.relative(SOURCE_DIR, filepath).split(path.sep);
	relativeFilepathArray.pop();

	return {
		RbxName: assetRbxName,
		RbxType: assetRbxType,
{'fraction_non_alphanumeric': 0.1406486814186117, 'fraction_numerical': 0.0007578053955744165, 'mean_word_length': 5.733673469387755, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4748043', 'n_tokens_mistral': 2553, 'n_tokens_neox': 2389, 'n_words': 404}