<commit_before>'use strict';

/**
 *
 * Usage
 *
 * node app/scripts/validate-pattern-data.js
 *  or
 * npm run lint:data
 *  or
 * yarn lint:data
 *
 * example config in ./config.json
 *
 *	"linter": {
 *		"patternData": {
 *			 "logMissingSchemaAsError": false,
 *			 "logMissingSchemaAsWarning": true
 *		}
 *	 }
 *
 */
const chalk = require('chalk');
const fs = require('fs');
const hbs = require('hbs');
const path = require('path');
const extend = require('extend');
const globby = require('globby');
const Ajv = require('ajv');
const config = require('../core/config');
const ajv = new Ajv({allErrors: true});
const patternBasePaths = Object.keys(config.nitro.patterns).map((key) => {
	return config.nitro.patterns[key].path;
});
const wildcard = '*';

const elementGlobs = patternBasePaths.map((patternBasePath) => {
	return `${patternBasePath}/${wildcard}`;
});

let logMissingSchemaAsError;
let logMissingSchemaAsWarning;
let errorCouter = 0;

if (config.linter && config.linter.patternData) {
	logMissingSchemaAsError = config.linter.patternData.logMissingSchemaAsError === undefined
		? false : config.linter.patternData.logMissingSchemaAsError;
	logMissingSchemaAsWarning = config.linter.patternData.logMissingSchemaAsWarning === undefined
		? true : config.linter.patternData.logMissingSchemaAsWarning;
}

globby.sync(elementGlobs).forEach((patternPath, index) => {
	const schemaFilePath = `${patternPath}/schema.json`;

	if (!fs.existsSync(schemaFilePath)) {
		if (logMissingSchemaAsError) {
			console.log(chalk.red(`Error (${patternPath}): no schema file found ⛔️`));
			errorCouter += 1;
			return true;
		}
		if (logMissingSchemaAsWarning) {
			console.log(chalk.yellow(`Warn (${patternPath}): no schema file found ⚠️`));
		}
		return true;
	}

	globby.sync([`${patternPath}/_data/${wildcard}.json`]).forEach((patternDataFilePath) => {
		const patternData = JSON.parse(fs.readFileSync(patternDataFilePath, 'utf8'));
		const schema = JSON.parse(fs.readFileSync(schemaFilePath, 'utf8'));

		const valid = ajv.validate(schema, patternData);
		if (!valid) {
			errorCouter += 1;
			console.log(chalk.red(`Error (${patternDataFilePath}): ${ajv.errorsText()} ⛔️`));
		}
	});
});

if (errorCouter <= 0) {
	console.log(chalk.green(`Success: all pattern data are valid! 👍`));
} else {
	process.abort();
}<commit_msg>nitro - #105 add subelements to pattern JSON-schema data validation task<commit_after>'use strict';

/**
 *
 * Usage
 *
 * node app/scripts/validate-pattern-data.js
 *  or
 * npm run lint:data
 *  or
 * yarn lint:data
 *
 * example config in ./config.json
 *
 *    "linter": {
 *        "patternData": {
 *            "logMissingSchemaAsError": false,
 *            "logMissingSchemaAsWarning": true
 *         }
 *    }
 *
 */
const chalk = require('chalk');
const fs = require('fs');
const path = require('path');
const globby = require('globby');
const Ajv = require('ajv');
const config = require('../core/config');
const ajv = new Ajv({allErrors: true});
const wildcard = '*';
const patternBasePaths = Object.keys(config.nitro.patterns).map((key) => {
	return config.nitro.patterns[key].path;
});
const patternGlobs = patternBasePaths.map((patternBasePath) => {
	return `${patternBasePath}/${wildcard}`;
}).concat(
	patternBasePaths.map((patternBasePath) => {
		return `${patternBasePath}/${wildcard}/elements/${wildcard}`;
	})
);

let logMissingSchemaAsError;
let logMissingSchemaAsWarning;
let errorCouter = 0;
let patternCouter = 0;

if (config.linter && config.linter.patternData) {
	logMissingSchemaAsError = config.linter.patternData.logMissingSchemaAsError === undefined
		? false : config.linter.patternData.logMissingSchemaAsError;
	logMissingSchemaAsWarning = config.linter.patternData.logMissingSchemaAsWarning === undefined
		? true : config.linter.patternData.logMissingSchemaAsWarning;
}

globby.sync(patternGlobs).forEach((patternPath, index) => {
	const schemaFilePath = `${patternPath}/schema.json`;
	patternCouter += 1;

	if (!fs.existsSync(schemaFilePath)) {
		if (logMissingSchemaAsError) {
			console.log(chalk.red(`Error (${patternPath}): no schema file found ⛔️`));
			errorCouter += 1;
			return true;
		}
		if (logMissingSchemaAsWarning) {
			console.log(chalk.yellow(`Warn (${patternPath}): no schema file found ⚠️`));
		}
		return true;
	}

	globby.sync([`${patternPath}/_data/${wildcard}.json`]).forEach((patternDataFilePath) => {
		const patternData = JSON.parse(fs.readFileSync(patternDataFilePath, 'utf8'));
		const schema = JSON.parse(fs.readFileSync(schemaFilePath, 'utf8'));

		const valid = ajv.validate(schema, patternData);
		if (!valid) {
			errorCouter += 1;
			console.log(chalk.red(`Error (${patternDataFilePath}): ${ajv.errorsText()} ⛔️`));
		}
	});
});

if (errorCouter <= 0) {
	console.log(chalk.green(`Success: all data from each of the ${patternCouter} patterns are valid! 👍`));
} else {
	process.abort();
}
{'fraction_non_alphanumeric': 0.16461916461916462, 'fraction_numerical': 0.003480753480753481, 'mean_word_length': 4.892641737032569, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6985966', 'n_tokens_mistral': 1680, 'n_tokens_neox': 1622, 'n_words': 311}