<commit_before>'use strict';

var path = require('path');

function loadCodePage(name) {
	var codePage;

	try {
		codePage = require(path.join(__dirname, 'pages', name));
	} catch(e) {
		throw new Error('Code page ' + name + ' not found');
	}

	return {
		findPage: findPage,
		findTag: findTag,
		get codePage() {
			return codePage;
		}
	};
}

function findPage(pageName) {
	if (pageName[pageName.length - 1] === ':') {
		pageName = pageName.substr(0, pageName.length - 1);
	}

	for (var i = 0; i < this.codePage.length; i++) {
		if (this.codePage[i].name === pageName) {
			return i;
		}
	}

	return null;
}

function findTag(pageNumber, searchFor) {
	var lookupAttr = typeof searchFor === 'string' ? 'name' : 'token';

	for (var i = 0; i < this.codePage[pageNumber].tags.length; i++) {
		if (this.codePage[pageNumber].tags[i][lookupAttr] === searchFor) {
			return this.codePage[pageNumber].tags[i];
		}
	}

	return null;
}

module.exports = loadCodePage;<commit_msg>Added methods to encode attributes and values<commit_after>'use strict';

var path = require('path');

function loadCodePage(name) {
	var codePage;

	try {
		codePage = require(path.join(__dirname, 'pages', name));
	} catch(e) {
		throw new Error('Code page ' + name + ' not found');
	}

	return {
		findPage: findPage,
		findTag: findTag,
		findAttributes : findAttributes,
		findValue : findValue,
		findMinIndexValue: findMinIndexValue,
		get codePage() {
			return codePage;
		}
	};
}

function findPage(pageName) {
	if (pageName[pageName.length - 1] === ':') {
		pageName = pageName.substr(0, pageName.length - 1);
	}

	for (var i = 0; i < this.codePage.length; i++) {
		if (this.codePage[i].name === pageName) {
			return i;
		}
	}

	return null;
}

function findTag(pageNumber, searchFor) {
	var lookupAttr = typeof searchFor === 'string' ? 'name' : 'token';

	for (var i = 0; i < this.codePage[pageNumber].tags.length; i++) {
		if (this.codePage[pageNumber].tags[i][lookupAttr] === searchFor) {
			return this.codePage[pageNumber].tags[i];
		}
	}

	return null;
}

function findAttributes(pageNumber, searchFor) {
	var attributes = []
	var lookupAttr = typeof searchFor === 'string' ? 'name' : 'token';

	for (var i = 0; i < this.codePage[pageNumber].attributes.length; i++) {
		if (this.codePage[pageNumber].attributes[i][lookupAttr] === searchFor) {
			attributes.push( this.codePage[pageNumber].attributes[i])
		}
	}
	if (attributes.length===0) return null
	else return attributes
}

function findValue(pageNumber, searchFor) {
	var lookupAttr = typeof searchFor === 'string' ? 'name' : 'token';
	for (var i = 0; i < this.codePage[pageNumber].values.length; i++) {
		if (this.codePage[pageNumber].values[i][lookupAttr] === searchFor) {
			return this.codePage[pageNumber].values[i]
		}
	}

	return null
}
/**
* Scans the list of values getting the minimun index of the searchFor element
*/
function findMinIndexValue(pageNumber, searchFor) {
	var lookupAttr = typeof searchFor === 'string' ? 'name' : 'token';
	var minIndexValue = null
	for (var i = 0; i < this.codePage[pageNumber].values.length; i++) {
		let index = searchFor.indexOf(this.codePage[pageNumber].values[i][lookupAttr])
		if ( index >= 0 &&
			(!minIndexValue || (index < minIndexValue.index))){
			minIndexValue = {index: index, value: this.codePage[pageNumber].values[i] }
		}
	}

	return minIndexValue
}
module.exports = loadCodePage;

{'fraction_non_alphanumeric': 0.15859421145894861, 'fraction_numerical': 0.004430005906674543, 'mean_word_length': 4.218798151001541, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1495605', 'n_tokens_mistral': 1249, 'n_tokens_neox': 1166, 'n_words': 275}