<commit_before>/*
	MIT License http://www.opensource.org/licenses/mit-license.php
	Author <NAME> @sokra
*/
"use strict";

const path = require("path");

const looksLikeAbsolutePath = (maybeAbsolutePath) => /^(?:[a-z]:\\|\/)/i.test(maybeAbsolutePath);

function makeRelative(compiler, identifier) {
	const context = compiler.context;
	return identifier.split("|")
		.map(str => str.split("!")
			.map(str => str.split(" ")
				.map(str => looksLikeAbsolutePath(str) ? path.relative(context, str) : str)
				.join(" "))
			.join("!"))
		.join("|");
}

class RecordIdsPlugin {

	apply(compiler) {
		compiler.plugin("compilation", compilation => {
			compilation.plugin("record-modules", (modules, records) => {
				if(!records.modules) records.modules = {};
				if(!records.modules.byIdentifier) records.modules.byIdentifier = {};
				if(!records.modules.usedIds) records.modules.usedIds = {};
				modules.forEach(function(module) {
					if(!module.portableId) module.portableId = makeRelative(compiler, module.identifier());
					const identifier = module.portableId;
					records.modules.byIdentifier[identifier] = module.id;
					records.modules.usedIds[module.id] = module.id;
				});
			});
			compilation.plugin("revive-modules", (modules, records) => {
				if(!records.modules) return;
				if(records.modules.byIdentifier) {
					const usedIds = {};
					modules.forEach(function(module) {
						if(module.id !== null) return;
						if(!module.portableId) module.portableId = makeRelative(compiler, module.identifier());
						const identifier = module.portableId;
						const id = records.modules.byIdentifier[identifier];
						if(id === undefined) return;
						if(usedIds[id]) return;
						usedIds[id] = true;
						module.id = id;
					});
				}
				compilation.usedModuleIds = records.modules.usedIds;
			});

			function getDepBlockIdent(chunk, block) {
				const ident = [];
				if(block.chunks.length > 1)
					ident.push(block.chunks.indexOf(chunk));
				while(block.parent) {
					const p = block.parent;
					const idx = p.blocks.indexOf(block);
					const l = p.blocks.length - 1;
					ident.unshift(`${idx}/${l}`);
					block = block.parent;
				}
				if(!block.identifier) return null;
				ident.unshift(makeRelative(compiler, block.identifier()));
				return ident.join(":");<commit_msg>use split trick to make code simpler, integrate makeRelative as static method into recordsidplugin<commit_after>/*
	MIT License http://www.opensource.org/licenses/mit-license.php
	Author <NAME> @sokra
*/
"use strict";

const path = require("path");

class RecordIdsPlugin {

	static looksLikeAbsolutePath(maybeAbsolutePath) {
		return /^(?:[a-z]:\\|\/)/i.test(maybeAbsolutePath);
	}

	static makeRelative(context, identifier) {
		return identifier
			.split(/([|! ])/)
			.map(str => RecordIdsPlugin.looksLikeAbsolutePath(str) ? path.relative(context, str) : str)
			.join("");
	}

	apply(compiler) {
		compiler.plugin("compilation", compilation => {
			compilation.plugin("record-modules", (modules, records) => {
				if(!records.modules) records.modules = {};
				if(!records.modules.byIdentifier) records.modules.byIdentifier = {};
				if(!records.modules.usedIds) records.modules.usedIds = {};
				modules.forEach(function(module) {
					if(!module.portableId) module.portableId = RecordIdsPlugin.makeRelative(compiler.context, module.identifier());
					const identifier = module.portableId;
					records.modules.byIdentifier[identifier] = module.id;
					records.modules.usedIds[module.id] = module.id;
				});
			});
			compilation.plugin("revive-modules", (modules, records) => {
				if(!records.modules) return;
				if(records.modules.byIdentifier) {
					const usedIds = {};
					modules.forEach(function(module) {
						if(module.id !== null) return;
						if(!module.portableId) module.portableId = RecordIdsPlugin.makeRelative(compiler.context, module.identifier());
						const identifier = module.portableId;
						const id = records.modules.byIdentifier[identifier];
						if(id === undefined) return;
						if(usedIds[id]) return;
						usedIds[id] = true;
						module.id = id;
					});
				}
				compilation.usedModuleIds = records.modules.usedIds;
			});

			function getDepBlockIdent(chunk, block) {
				const ident = [];
				if(block.chunks.length > 1)
					ident.push(block.chunks.indexOf(chunk));
				while(block.parent) {
					const p = block.parent;
					const idx = p.blocks.indexOf(block);
					const l = p.blocks.length - 1;
					ident.unshift(`${idx}/${l}`);
					block = block.parent;
				}
				if(!block.identifier) return null;
				ident.unshift(RecordIdsPlugin.makeRelative(compiler.context, block.identifier()));
				return ident.join(":");
{'fraction_non_alphanumeric': 0.15887251761691223, 'fraction_numerical': 0.0008541533205210335, 'mean_word_length': 3.707537688442211, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1512889', 'n_tokens_mistral': 1757, 'n_tokens_neox': 1428, 'n_words': 252}