<commit_before>"use strict";

// load configuration
let conf;
try {
	conf = require("../../build.user.conf");
} catch(e) {
	conf = require("../../build.conf");
}

const load = (path) => {
	try {
		return require(path);
	} catch(e) {
		console.log(`Couldn't load module "${path}", try running "npm install"`);
		throw e;
		process.exit();
	}
};

const gutil = load("gulp-util");

// pull the process arguments we care about out
let args = [];

for (let i = 2; i < process.argv.length; i++) {
	let arg = process.argv[i];

	if (arg.startsWith("--")) {
		args = process.argv.slice(i);
		break;
	}
}

// black list: don't build these modules
// white list: only build these modules
// gray list: build these modules despite defaults
let params = {
	white: null,
	gray: null,
	black: null
};

const getValue = (name, base, core) => {
	let conf = core.conf;

	if (params[name] != null) {
		return params[name];
	}

	if (core.preset[name] != null) {
		return core.preset[name];
	}

	if (conf[name] != null) {
		return conf[name];
	}

	return base;
};

const core = {
	version: "2.0.0",

	modules: conf.modules || ["browser", "server", "styles", "static"],
	buildTasks: [],
	browserSync: null,

	conf: conf,
	params: params,

	get: (key) => getValue(key, null, core),

	get preset() {
		let name = params.preset || conf.preset;

		if (!conf.presets[name]) {
			console.error(`Couldn't find preset '${name}'!`);
			process.exit(1);
			return;
		}

		return conf.presets[name];
	},

	get watch() {
		return getValue("watch", false, core);
	},

	get notify() {
		return getValue("notify", false, core);
	},

	get sourcemaps() {
		return getValue("sourcemaps", true, core);
	},

	get minify() {
		return getValue("minify", false, core);
	},

	get out() {
		return getValue("out", "out", core);
	},

	get useBrowsersync() {
		return getValue("browsersync", !!(core.preset.browserSync || core.conf.browserSync), core);
	},

	load: load,

	getName(transform) {
		if (transform.name) {
			return transform.name;
		}

		return transform.type.charAt(0).toUpperCase() + transform.type.slice(1);
	},

	getTransforms(type) {
		let params = core.params;
		let unfiltered = [...core.conf.transforms];

		if (core.preset.transforms) {
			unfiltered.push(...core.preset.transforms);
		}

		let byID = new Map();
		let unID = [];

		for (let transform of unfiltered) {
			if (transform.id != null) {
				let existing = byID.get(transform.id);

				if (existing) {
					Object.assign(existing, transform);
				} else {
					byID.set(transform.id, transform);
				}
			} else {
				unID.push(transform);
			}
		}

		let transforms = unID.slice();

		for (let pair of byID) {
			let transform = pair[1];
			transforms.push(transform);
		}

		return transforms.filter(transform => {
			let name = (transform.name || transform.type || "unnamed").toLowerCase();

			if (type != null && transform.type !== type) {
				return false;
			}

			if (params.black && params.black[name]) {
				return false;
			}

			if (params.white && !params.white[name]) {
				return false;
			}

			if (transform.disabled) {
				if (params.gray) {
					if (!params.gray[name]) {
						return false;
					}
				} else {
					return false;
				}
			}

			return true;
		});
	},

	onError(err) {
		if (core.notify) {
			require("gulp-notify").onError("Error: <%= error.message %>")(err);
		} else {
			gutil.log(gutil.colors.red("Build error!"));
			gutil.log(err.message);
		}

		this.emit("end");
	},

	getErrorHandler: function(name) {
		return function(err) {
			if (core.notify) {
				require("gulp-notify").onError("Error: <%= error.message %>")(err);
			} else {
				gutil.log(gutil.colors.red(name + ": Build error!"));
				gutil.log(err.message);
				gutil.log(err.stack);
			}

			this.emit("end");
		};
	},

	getNotify(msg) {
		let notify = require("gulp-notify");

		let notifier;

		if (!core.notify) {
			notifier = (options, callback) => callback();
		}

		return notify({
			message: msg,
			onLast: true,
			notifier: notifier
		});
	}
};

{
	const modes = {
		white: true,
		gray: true,
		black: true
	};

	const flags = {
		watch: true,
		production: true,
		sourcemaps: true,
		minify: true,
		notify: true,
		browsersync: true
	};

	const settings = {
		preset: true,
		out: true
	};

	const aliases = {
		"only": "white",
		"except": "black"
	};

	const toBoolean = (v) => {
		v = v.toLowerCase();

		if (v === "true" || v === "yes") {
			return true;
		}

		if (v === "false" || v === "no") {
			return false;
		}

		return true;
	};

	for (let arg of args) {
		if (!arg.startsWith("--")) {
			console.error(`Unknown command line option '${arg}'`);
			process.exit(1);
		}

		arg = arg.slice(2).toLowerCase();
		arg = aliases[arg] || arg;

		let setter = arg.match(/^([^=]+)=(.+)/);

		if (!setter) {
			if (flags[arg]) {
				params[arg] = true;
				continue;
			}

			console.error(`Unknown command line option '${arg}'`);
			process.exit(1);
		}

		let key = setter[1];
		let value = setter[2];

		key = aliases[key] || key;

		if (modes[key]) {
			if (params[key] == null) {
				params[key] = {};
			}

			let list = value.split(",");
			for (let e of list) {
				params[key][e.toLowerCase()] = true;
			}

			continue;
		}


		if (flags[key]) {
			params[key] = toBoolean(value);
			continue;
		}

		if (settings[key]) {
			params[key] = value;
			continue;
		}
	}
}

module.exports = core;<commit_msg>Change core to match against id first<commit_after>"use strict";

// load configuration
let conf;
try {
	conf = require("../../build.user.conf");
} catch(e) {
	conf = require("../../build.conf");
}

const load = (path) => {
	try {
		return require(path);
	} catch(e) {
		console.log(`Couldn't load module "${path}", try running "npm install"`);
		throw e;
		process.exit();
	}
};

const gutil = load("gulp-util");

// pull the process arguments we care about out
let args = [];

for (let i = 2; i < process.argv.length; i++) {
	let arg = process.argv[i];

	if (arg.startsWith("--")) {
		args = process.argv.slice(i);
		break;
	}
}

// black list: don't build these modules
// white list: only build these modules
// gray list: build these modules despite defaults
let params = {
	white: null,
	gray: null,
	black: null
};

const getValue = (name, base, core) => {
	let conf = core.conf;

	if (params[name] != null) {
		return params[name];
	}

	if (core.preset[name] != null) {
		return core.preset[name];
	}

	if (conf[name] != null) {
		return conf[name];
	}

	return base;
};

const core = {
	version: "2.0.0",

	modules: conf.modules || ["browser", "server", "styles", "static"],
	buildTasks: [],
	browserSync: null,

	conf: conf,
	params: params,

	get: (key) => getValue(key, null, core),

	get preset() {
		let name = params.preset || conf.preset;

		if (!conf.presets[name]) {
			console.error(`Couldn't find preset '${name}'!`);
			process.exit(1);
			return;
		}

		return conf.presets[name];
	},

	get watch() {
		return getValue("watch", false, core);
	},

	get notify() {
		return getValue("notify", false, core);
	},

	get sourcemaps() {
		return getValue("sourcemaps", true, core);
	},

	get minify() {
		return getValue("minify", false, core);
	},

	get out() {
		return getValue("out", "out", core);
	},

	get useBrowsersync() {
		return getValue("browsersync", !!(core.preset.browserSync || core.conf.browserSync), core);
	},

	load: load,

	getName(transform) {
		if (transform.name) {
			return transform.name;
		}

		return transform.type.charAt(0).toUpperCase() + transform.type.slice(1);
	},

	getTransforms(type) {
		let params = core.params;
		let unfiltered = [...core.conf.transforms];

		if (core.preset.transforms) {
			unfiltered.push(...core.preset.transforms);
		}

		let byID = new Map();
		let unID = [];

		for (let transform of unfiltered) {
			if (transform.id != null) {
				let existing = byID.get(transform.id);

				if (existing) {
					Object.assign(existing, transform);
				} else {
					byID.set(transform.id, transform);
				}
			} else {
				unID.push(transform);
			}
		}

		let transforms = unID.slice();

		for (let pair of byID) {
			let transform = pair[1];
			transforms.push(transform);
		}

		return transforms.filter(transform => {
			let name = (transform.id || transform.name || transform.type || "unnamed").toLowerCase();

			if (type != null && transform.type !== type) {
				return false;
			}

			if (params.black && params.black[name]) {
				return false;
			}

			if (params.white && !params.white[name]) {
				return false;
			}

			if (transform.disabled) {
				if (params.gray) {
					if (!params.gray[name]) {
						return false;
					}
				} else {
					return false;
				}
			}

			return true;
		});
	},

	onError(err) {
		if (core.notify) {
			require("gulp-notify").onError("Error: <%= error.message %>")(err);
		} else {
			gutil.log(gutil.colors.red("Build error!"));
			gutil.log(err.message);
		}

		this.emit("end");
	},

	getErrorHandler: function(name) {
		return function(err) {
			if (core.notify) {
				require("gulp-notify").onError("Error: <%= error.message %>")(err);
			} else {
				gutil.log(gutil.colors.red(name + ": Build error!"));
				gutil.log(err.message);
				gutil.log(err.stack);
			}

			this.emit("end");
		};
	},

	getNotify(msg) {
		let notify = require("gulp-notify");

		let notifier;

		if (!core.notify) {
			notifier = (options, callback) => callback();
		}

		return notify({
			message: msg,
			onLast: true,
			notifier: notifier
		});
	}
};

{
	const modes = {
		white: true,
		gray: true,
		black: true
	};

	const flags = {
		watch: true,
		production: true,
		sourcemaps: true,
		minify: true,
		notify: true,
		browsersync: true
	};

	const settings = {
		preset: true,
		out: true
	};

	const aliases = {
		"only": "white",
		"except": "black"
	};

	const toBoolean = (v) => {
		v = v.toLowerCase();

		if (v === "true" || v === "yes") {
			return true;
		}

		if (v === "false" || v === "no") {
			return false;
		}

		return true;
	};

	for (let arg of args) {
		if (!arg.startsWith("--")) {
			console.error(`Unknown command line option '${arg}'`);
			process.exit(1);
		}

		arg = arg.slice(2).toLowerCase();
		arg = aliases[arg] || arg;

		let setter = arg.match(/^([^=]+)=(.+)/);

		if (!setter) {
			if (flags[arg]) {
				params[arg] = true;
				continue;
			}

			console.error(`Unknown command line option '${arg}'`);
			process.exit(1);
		}

		let key = setter[1];
		let value = setter[2];

		key = aliases[key] || key;

		if (modes[key]) {
			if (params[key] == null) {
				params[key] = {};
			}

			let list = value.split(",");
			for (let e of list) {
				params[key][e.toLowerCase()] = true;
			}

			continue;
		}


		if (flags[key]) {
			params[key] = toBoolean(value);
			continue;
		}

		if (settings[key]) {
			params[key] = value;
			continue;
		}
	}
}

module.exports = core;
{'fraction_non_alphanumeric': 0.17671280883125987, 'fraction_numerical': 0.002277904328018223, 'mean_word_length': 2.4054295942720763, 'pattern_counts': {'":': 6, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6583746', 'n_tokens_mistral': 4741, 'n_tokens_neox': 3959, 'n_words': 914}