<commit_before>'use strict';

const unicode = require('unicode-8.0.0');
const propertyAliases = require('unicode-property-aliases');
const valueAliases = require('unicode-property-value-aliases');
const jsesc = require('jsesc');

const normalize = function(string) {
	const normalized = string
		// Remove case distinctions.
		.toLowerCase()
		// Remove whitespace.
		.replace(/\s/g, '')
		// Remove `-` and `_`.
		.replace(/[-_]/g, '');
	// Note: don’t remove `is` prefixes, since that would mangle
	// `isc`/`ISO_Comment`.
	return normalized;
};

const aliasToProperty = new Map();
const propertyToValueAliases = new Map();
for (const [propertyAlias, canonicalProperty] of propertyAliases.entries()) {
	// Map the normalized version of this property alias to the canonical
	// property name.
	const normalizedPropertyAlias = normalize(propertyAlias);
	aliasToProperty.set(normalizedPropertyAlias, canonicalProperty);
	// Map the normalized version of the canonical property name to the canonical
	// property name.
	const normalizedCanonicalProperty = normalize(canonicalProperty);
	aliasToProperty.set(normalizedCanonicalProperty, canonicalProperty);
	// Handle the property value aliases for this property.
	const valueAliasMappings = valueAliases.get(canonicalProperty);
	// { property value alias => canonical property value }
	const aliasToValue = new Map();
	if (valueAliasMappings) {
		for (const [valueAlias, canonicalValue] of valueAliasMappings.entries()) {
			const normalizedValueAlias = normalize(valueAlias);
			aliasToValue.set(normalizedValueAlias, canonicalValue);
			const normalizedCanonicalValue = normalize(canonicalValue);
			aliasToValue.set(normalizedCanonicalValue, canonicalValue);
		}
	}
	// Add normalized “aliases” for known blocks.
	if (canonicalProperty == 'Block') {
		for (const block of unicode.blocks) {
			const normalizedBlock = normalize(block);
			aliasToValue.set(normalizedBlock, block);
		}
	}
	// Add normalized “aliases” for known categories.
	if (canonicalProperty == 'General_Category') {
		for (const category of unicode.categories) {
			const normalizedCategory = normalize(category);
			const canonicalCategory = valueAliasMappings.get(category);
			aliasToValue.set(normalizedCategory, canonicalCategory);
		}
	}
	// Add normalized “aliases” for known scripts.
	if (canonicalProperty == 'Script') {
		for (const script of unicode.scripts) {
			const normalizedScript = normalize(script);
			aliasToValue.set(normalizedScript, script);
		}
	}
	// Add normalized “aliases” for known script extensions.
	if (canonicalProperty == 'Script_Extensions') {
		for (const script of unicode['script-extensions']) {
			const normalizedScript = normalize(script);
			aliasToValue.set(normalizedScript, script);
		}
	}
	// { canonical property => { property value alias => canonical property value } }
	if (aliasToValue.size) {
		propertyToValueAliases.set(canonicalProperty, aliasToValue);
	} else {
		console.log(`No property value aliases for ${ canonicalProperty }`);
	}
}

// Add normalized “aliases” for binary properties.
for (const property of unicode.properties) {
	const normalizedProperty = normalize(property);
	aliasToProperty.set(normalizedProperty, property);
}<commit_msg>Simplify build script<commit_after>'use strict';

const unicode = require('unicode-8.0.0');
const propertyAliases = require('unicode-property-aliases');
const valueAliases = require('unicode-property-value-aliases');
const jsesc = require('jsesc');

const normalize = function(string) {
	const normalized = string
		// Remove case distinctions.
		.toLowerCase()
		// Remove whitespace.
		.replace(/\s/g, '')
		// Remove `-` and `_`.
		.replace(/[-_]/g, '');
	// Note: don’t remove `is` prefixes, since that would mangle
	// `isc`/`ISO_Comment`.
	return normalized;
};

const aliasToProperty = new Map();
const propertyToValueAliases = new Map();
for (const [propertyAlias, canonicalProperty] of propertyAliases.entries()) {
	// Map the normalized version of this property alias to the canonical
	// property name.
	const normalizedPropertyAlias = normalize(propertyAlias);
	aliasToProperty.set(normalizedPropertyAlias, canonicalProperty);
	// Map the normalized version of the canonical property name to the canonical
	// property name.
	const normalizedCanonicalProperty = normalize(canonicalProperty);
	aliasToProperty.set(normalizedCanonicalProperty, canonicalProperty);
	// Handle the property value aliases for this property.
	const valueAliasMappings = valueAliases.get(canonicalProperty);
	// { property value alias => canonical property value }
	const aliasToValue = new Map();
	if (valueAliasMappings) {
		for (const [valueAlias, canonicalValue] of valueAliasMappings.entries()) {
			const normalizedValueAlias = normalize(valueAlias);
			aliasToValue.set(normalizedValueAlias, canonicalValue);
			const normalizedCanonicalValue = normalize(canonicalValue);
			aliasToValue.set(normalizedCanonicalValue, canonicalValue);
		}
	}
	// Add normalized “aliases” for known blocks.
	if (canonicalProperty == 'Block') {
		for (const block of unicode.blocks) {
			const normalizedBlock = normalize(block);
			aliasToValue.set(normalizedBlock, block);
		}
	}
	// Add normalized “aliases” for known scripts.
	if (canonicalProperty == 'Script') {
		for (const script of unicode.scripts) {
			const normalizedScript = normalize(script);
			aliasToValue.set(normalizedScript, script);
		}
	}
	// Add normalized “aliases” for known script extensions.
	if (canonicalProperty == 'Script_Extensions') {
		for (const script of unicode['script-extensions']) {
			const normalizedScript = normalize(script);
			aliasToValue.set(normalizedScript, script);
		}
	}
	// { canonical property => { property value alias => canonical property value } }
	if (aliasToValue.size) {
		propertyToValueAliases.set(canonicalProperty, aliasToValue);
	} else {
		console.log(`No property value aliases for ${ canonicalProperty }`);
	}
}

// Add normalized “aliases” for binary properties.
for (const property of unicode.properties) {
	const normalizedProperty = normalize(property);
	aliasToProperty.set(normalizedProperty, property);
}
{'fraction_non_alphanumeric': 0.11283221914234469, 'fraction_numerical': 0.0009783140388064568, 'mean_word_length': 5.031465093411996, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2182229', 'n_tokens_mistral': 1800, 'n_tokens_neox': 1667, 'n_words': 474}