<commit_before>'use strict';

var path = require('path');
var fs = require('fs');

var async = require('async');
var glob = require('glob');
var chalk = require('chalk');

module.exports = exports = checkLocales;

var templateKeyPattern = /\{@pre\s+type="content"\s+key="([a-zA-Z0-9.\[\]]+)"(?:\s+mode="([^"]+)")?\s*\/\}/gm;
var templatesExt = '.dust';

var bundleKeyPattern = /^\s*([a-zA-Z0-9.\[\]]+)\s*=.*$/gm;
var bundlesExt = '.properties';

function checkLocales(root, callback) {
	var templatesPath = path.join(root, 'public', 'templates');
	var bundlesPath = path.join(root, 'locales');

	var templates = {};
	var locales = [];
	var badBundles = [];

	glob(path.join(templatesPath, '**', '*' + templatesExt), loadTemplates);

	function loadTemplates(error, filenames) {
		if (error) {
			callback(error);
			return;
		}

		async.each(filenames, loadTemplate, onTemplatesLoaded);
	}

	function loadTemplate(filename, callback) {
		fs.readFile(filename, 'utf8', parseTemplate);

		function parseTemplate(error, file) {
			if (error) {
				callback(error);
				return;
			}

			var name = path.relative(templatesPath, filename).slice(0, -templatesExt.length);

			templates[name] = {
				name: name,
				keys: getTemplateKeys(file),
				locales: {}
			};

			callback(null);
		}
	}

	function onTemplatesLoaded(error) {
		if (error) {
			callback(error);
			return;
		}

		glob(path.join(bundlesPath, '*', '*'), loadLocales);
	}

	function loadLocales(error, dirnames) {
		if (error) {
			callback(error);
			return;
		}

		async.each(dirnames, loadLocale, onLocalesLoaded);
	}

	function loadLocale(dirname, callback) {
		var locale = path.relative(bundlesPath, dirname);
		locales.push(locale);

		glob(path.join(dirname, '**', '*' + bundlesExt), loadBundles);

		function loadBundles(error, filenames) {
			if (error) {
				callback(error);
				return;
			}

			async.each(filenames, loadBundle, callback);
		}

		function loadBundle(filename, callback) {
			fs.readFile(filename, 'utf8', parseBundle);

			function parseBundle(error, file) {
				if (error) {
					callback(error);
					return;
				}

				var name = path.relative(dirname, filename).slice(0, -bundlesExt.length);

				if (!templates.hasOwnProperty(name)) {
					onBadBundle('unused', path.relative(bundlesPath, filename));<commit_msg>Add an ignore option<commit_after>'use strict';

var path = require('path');
var fs = require('fs');

var async = require('async');
var glob = require('glob');
var chalk = require('chalk');

module.exports = exports = checkLocales;

var templateKeyPattern = /\{@pre\s+type="content"\s+key="([a-zA-Z0-9.\[\]]+)"(?:\s+mode="([^"]+)")?\s*\/\}/gm;
var templatesExt = '.dust';

var bundleKeyPattern = /^\s*([a-zA-Z0-9.\[\]]+)\s*=.*$/gm;
var bundlesExt = '.properties';

function checkLocales(root, options, callback) {
	if (typeof options === 'function') {
		callback = options;
		options = {};

	} else if (!(options instanceof Object)) {
		options = {};
	}

	var templatesPath = path.join(root, 'public', 'templates');
	var bundlesPath = path.join(root, 'locales');

	var globOptions = {
		ignore: options.ignore || []
	};

	var templates = {};
	var locales = [];
	var badBundles = [];

	glob(path.join(templatesPath, '**', '*' + templatesExt), globOptions, loadTemplates);

	function loadTemplates(error, filenames) {
		if (error) {
			callback(error);
			return;
		}

		async.each(filenames, loadTemplate, onTemplatesLoaded);
	}

	function loadTemplate(filename, callback) {
		fs.readFile(filename, 'utf8', parseTemplate);

		function parseTemplate(error, file) {
			if (error) {
				callback(error);
				return;
			}

			var name = path.relative(templatesPath, filename).slice(0, -templatesExt.length);

			templates[name] = {
				name: name,
				keys: getTemplateKeys(file),
				locales: {}
			};

			callback(null);
		}
	}

	function onTemplatesLoaded(error) {
		if (error) {
			callback(error);
			return;
		}

		glob(path.join(bundlesPath, '*', '*'), globOptions, loadLocales);
	}

	function loadLocales(error, dirnames) {
		if (error) {
			callback(error);
			return;
		}

		async.each(dirnames, loadLocale, onLocalesLoaded);
	}

	function loadLocale(dirname, callback) {
		var locale = path.relative(bundlesPath, dirname);
		locales.push(locale);

		glob(path.join(dirname, '**', '*' + bundlesExt), globOptions, loadBundles);

		function loadBundles(error, filenames) {
			if (error) {
				callback(error);
				return;
			}

			async.each(filenames, loadBundle, callback);
		}

		function loadBundle(filename, callback) {
			fs.readFile(filename, 'utf8', parseBundle);

			function parseBundle(error, file) {
				if (error) {
					callback(error);
					return;
				}

				var name = path.relative(dirname, filename).slice(0, -bundlesExt.length);

				if (!templates.hasOwnProperty(name)) {
					onBadBundle('unused', path.relative(bundlesPath, filename));
{'fraction_non_alphanumeric': 0.17457383446292873, 'fraction_numerical': 0.003286095707537482, 'mean_word_length': 4.067637877211238, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5020073', 'n_tokens_mistral': 1898, 'n_tokens_neox': 1622, 'n_words': 323}