<commit_before>var fs = require('fs');
var Mod = require("./../common/mod");
var RepoServers = require('./../common/reposervers');
var basename_override = JSON.parse(fs.readFileSync("data/basename_override.json", 'utf8'));
var url_cachefile = JSON.parse(fs.readFileSync("data/url_result_cache.json", 'utf8'));

// Blacklists and verified lists
var link_blacklist = JSON.parse(fs.readFileSync("data/blacklist.json", 'utf8'));
var link_verified = JSON.parse(fs.readFileSync("data/verified.json", 'utf8'));
function linkIsBlacklisted(link) {
	for (var j = 0; j < link_blacklist.length; j++) {
		if (link.indexOf(link_blacklist[j]) >= 0) {
			return true;
		}
	}
	return false;
}

function linkIsVerified(link) {
	for (var j = 0; j < link_verified.length; j++) {
		if (link.indexOf(link_blacklist[j]) >= 0) {
			return true;
		}
	}
	return false;
}

function addMod(stats, author, details) {
	author = author || "error";
	author = author.toLowerCase();
	if (details.constructor != Object) {
		console.log(author);
		console.log(details);
		throw("Details is not an object!");
	}

	var a = stats.out_errors[author];
	if (!a) {
		a = [];
		stats.out_errors[author] = a;
	}
	a.push(details);
}

function addModReason(stats, author, mod, reason) {
	var res = {};
	for (var key in mod) {
		if (mod.hasOwnProperty(key)) {
			res[key] = mod[key];
		}
	}
	res.msg = reason;
	addMod(stats, author, res);
}

function addSuccess(stats, author, mod) {
	addModReason(stats, author, mod, "added");
}

console.log("Creating repo servers...");
var reposervers = [];
reposervers.push(new (RepoServers.GithubRepoServer)("data/cache_github.json"));

function getInfoFromModLink(stats, mod, link) {
	return new Promise(function(resolve, reject) {
		if (!link || link.length == 0) {
			reject("needs download link");
			return;
		}

		for (var i = 0; i < reposervers.length; i++) {
			var reposerver = reposervers[i];
			var repo = reposerver.getRepoFromURL(link);
			if (repo) {
				var sname = reposerver.getServerName();
				var cell = stats.source[sname]
				stats.source[sname] = (cell) ? cell + 1 : 1
				mod.repo_host = reposerver.getServerName();
				mod.repo = reposerver.getRepoURL(repo);
				mod.repo_author = repo.user;
				mod.repo_name = repo.repo;
				reposerver.getAllInfo(repo, mod).then(resolve).catch(reject);
				return;
			}
		}

		resolve();
	});
}

function processMod(stats, json) {
	return new Promise(function(resolve, reject) {
		var mod = new Mod(json.author);
		mod.type = json.type;
		mod.forum_url = "https://forum.minetest.net/viewtopic.php?t=" + json.topicId;
		mod.extractInfoFromTitle(json.title);

		var bn_or = basename_override[json.link];
		if (bn_or) {
			mod.basename = bn_or;
		}

		if (linkIsBlacklisted(json.link)) {
			addModReason(stats, mod.author, mod, "download link is blacklisted");
			reject("download link is blacklisted");
		} else {
			getInfoFromModLink(stats, mod, json.link).then(function() {
				if (linkIsBlacklisted(json.link)) {
					addModReason(stats, mod.author, mod, "download link is blacklisted");
					reject("download link is blacklisted");
				} else {
					var problem = mod.getFirstProblem();
					if (problem) {
						addModReason(stats, mod.author, mod, problem);
						reject(problem);
					} else {
						if (mod.download_link.indexOf(mod.basename) < 0) {
							stats.c_potwrong++;
						}

						var url_cache = url_cachefile[mod.download_link];
						if (url_cache) {
							if (url_cache.status == 200) {
								this.download_size = url_cache.size;
							} else if (url_cache.status == -1) {
								reject("download does not result in zip file");
								return;
							} else {
								reject("download does not lead to an existent URL");
								return;
							}
						} else {
							reject("download unknown (run check_urls!)");
							return;
						}

						addSuccess(stats, mod.author, mod);
						resolve(mod);
					}
				}
			}).catch(function(e) {
				addModReason(stats, mod.author, mod, e);
				reject(e);
			});
		}
	});
}

function processAllMods(stats, json_array) {
	return new Promise(function(resolve, reject) {
		var waiting_for = 0;
		var res = [];
		var idx = 0;

		var int_id = setInterval(function() {
			var num_spawned = 0;
			for (var i = 0; i < 100 && idx < json_array.length && waiting_for < 100; i++) {
				waiting_for++;
				processMod(stats, json_array[idx]).then(function(mod) {<commit_msg>Add more modReasons<commit_after>var fs = require('fs');
var Mod = require("./../common/mod");
var RepoServers = require('./../common/reposervers');
var basename_override = JSON.parse(fs.readFileSync("data/basename_override.json", 'utf8'));
var url_cachefile = JSON.parse(fs.readFileSync("data/url_result_cache.json", 'utf8'));

// Blacklists and verified lists
var link_blacklist = JSON.parse(fs.readFileSync("data/blacklist.json", 'utf8'));
var link_verified = JSON.parse(fs.readFileSync("data/verified.json", 'utf8'));
function linkIsBlacklisted(link) {
	for (var j = 0; j < link_blacklist.length; j++) {
		if (link.indexOf(link_blacklist[j]) >= 0) {
			return true;
		}
	}
	return false;
}

function linkIsVerified(link) {
	for (var j = 0; j < link_verified.length; j++) {
		if (link.indexOf(link_blacklist[j]) >= 0) {
			return true;
		}
	}
	return false;
}

function addMod(stats, author, details) {
	author = author || "error";
	author = author.toLowerCase();
	if (details.constructor != Object) {
		console.log(author);
		console.log(details);
		throw("Details is not an object!");
	}

	var a = stats.out_errors[author];
	if (!a) {
		a = [];
		stats.out_errors[author] = a;
	}
	a.push(details);
}

function addModReason(stats, author, mod, reason) {
	var res = {};
	for (var key in mod) {
		if (mod.hasOwnProperty(key)) {
			res[key] = mod[key];
		}
	}
	res.msg = reason;
	addMod(stats, author, res);
}

function addSuccess(stats, author, mod) {
	addModReason(stats, author, mod, "added");
}

console.log("Creating repo servers...");
var reposervers = [];
reposervers.push(new (RepoServers.GithubRepoServer)("data/cache_github.json"));

function getInfoFromModLink(stats, mod, link) {
	return new Promise(function(resolve, reject) {
		if (!link || link.length == 0) {
			reject("needs download link");
			return;
		}

		for (var i = 0; i < reposervers.length; i++) {
			var reposerver = reposervers[i];
			var repo = reposerver.getRepoFromURL(link);
			if (repo) {
				var sname = reposerver.getServerName();
				var cell = stats.source[sname]
				stats.source[sname] = (cell) ? cell + 1 : 1
				mod.repo_host = reposerver.getServerName();
				mod.repo = reposerver.getRepoURL(repo);
				mod.repo_author = repo.user;
				mod.repo_name = repo.repo;
				reposerver.getAllInfo(repo, mod).then(resolve).catch(reject);
				return;
			}
		}

		resolve();
	});
}

function processMod(stats, json) {
	return new Promise(function(resolve, reject) {
		var mod = new Mod(json.author);
		mod.type = json.type;
		mod.forum_url = "https://forum.minetest.net/viewtopic.php?t=" + json.topicId;
		mod.extractInfoFromTitle(json.title);

		var bn_or = basename_override[json.link];
		if (bn_or) {
			mod.basename = bn_or;
		}

		if (linkIsBlacklisted(json.link)) {
			addModReason(stats, mod.author, mod, "download link is blacklisted");
			reject("download link is blacklisted");
		} else {
			getInfoFromModLink(stats, mod, json.link).then(function() {
				if (linkIsBlacklisted(json.link)) {
					addModReason(stats, mod.author, mod, "download link is blacklisted");
					reject("download link is blacklisted");
				} else {
					var problem = mod.getFirstProblem();
					if (problem) {
						addModReason(stats, mod.author, mod, problem);
						reject(problem);
					} else {
						if (mod.download_link.indexOf(mod.basename) < 0) {
							stats.c_potwrong++;
						}

						var url_cache = url_cachefile[mod.download_link];
						if (url_cache) {
							if (url_cache.status == 200) {
								this.download_size = url_cache.size;
							} else if (url_cache.status == -1) {
								addModReason(stats, mod.author, mod, "download does not result in a zip file");
								reject("download does not result in zip file");
								return;
							} else {
								addModReason(stats, mod.author, mod, "download does not lead to a existent URL");
								reject("download does not lead to an existent URL");
								return;
							}
						} else {
							addModReason(stats, mod.author, mod, "download unknown (this should never happen)");
							reject("download unknown (run check_urls!)");
							return;
						}

						addSuccess(stats, mod.author, mod);
						resolve(mod);
					}
				}
			}).catch(function(e) {
				addModReason(stats, mod.author, mod, e);
				reject(e);
			});
		}
	});
}

function processAllMods(stats, json_array) {
	return new Promise(function(resolve, reject) {
		var waiting_for = 0;
		var res = [];
		var idx = 0;

		var int_id = setInterval(function() {
			var num_spawned = 0;
			for (var i = 0; i < 100 && idx < json_array.length && waiting_for < 100; i++) {
				waiting_for++;
				processMod(stats, json_array[idx]).then(function(mod) {
{'fraction_non_alphanumeric': 0.15279929187873423, 'fraction_numerical': 0.005974773179907059, 'mean_word_length': 3.497014925373134, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 25, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5797185', 'n_tokens_mistral': 3709, 'n_tokens_neox': 3095, 'n_words': 697}