<commit_before>/*!
 * A Grunt checker for the 'banana' format JSON i18n message files.
 */

/* eslint-env node */
module.exports = function ( grunt ) {
	grunt.registerMultiTask( 'banana', function () {
		var ok,
			path = require( 'path' ),
			fs = require( 'fs' ),
			options = this.options( {
				sourceFile: 'en.json',
				documentationFile: 'qqq.json',

				requireMetadata: true,

				requireCompleteMessageDocumentation: true,
				disallowUnusedDocumentation: true,

				disallowBlankTranslations: true,
				disallowDuplicateTranslations: false,
				disallowUnusedTranslations: false,

				requireLowerCase: true,
				requireKeyPrefix: [],

				requireCompleteTranslationLanguages: [],
				requireCompleteTranslationMessages: [],

				skipIncompleteMessageDocumentation: []
			} ),
			messageCount = 0;

		if ( this.filesSrc.length === 0 ) {
			grunt.log.error( 'Target directory does not exist.' );
			return false;
		}

		ok = true;

		this.filesSrc.forEach( function ( dir ) {
			var message, index, offset,
				// Source message data
				sourceMessages, sourceMessageKeys,
				// Documentation message data
				documentationMessages, documentationMessageKeys,
				// Translated message data
				translatedFiles,
				jsonFilenameRegex = /(.*)\.json$/,
				translatedData = {},
				documentationMessageBlanks = [],
				sourceMessageMissing = [],
				sourceMessageWrongCase = [],
				sourceMessageWrongPrefix = [],
				count = 0;

			function messages( filename, type ) {
				var messageArray;

				try {
					messageArray = grunt.file.readJSON( path.resolve( dir, filename ) );
				} catch ( e ) {
					grunt.log.error( 'Loading ' + type + ' messages failed: "' + e + '".' );
					ok = false;
					throw e;
				}

				return messageArray;
			}

			function keysNoMetadata( messageArray, type ) {
				var keys, offset;

				try {
					keys = Object.keys( messageArray );
				} catch ( e ) {
					grunt.log.error( 'Loading ' + type + ' messages failed: "' + e + '".' );
					ok = false;
					throw e;
				}

				offset = keys.indexOf( '@metadata' );
				if ( offset === -1 ) {
					if ( options.requireMetadata ) {
						grunt.log.error( 'No metadata block in the ' + type + ' messages file.' );
						ok = false;
					}
				} else {
					keys.splice( offset, 1 );
				}

				return keys;
			}

			sourceMessages = messages( options.sourceFile, 'source' );
			sourceMessageKeys = keysNoMetadata( sourceMessages, 'source' );

			documentationMessages = messages( options.documentationFile, 'documentation' );
			documentationMessageKeys = keysNoMetadata( documentationMessages, 'documentation' );

			// Count after @metadata is removed
			messageCount += sourceMessageKeys.length;

			translatedFiles = fs.readdirSync( dir ).filter( function ( value ) {
				return (
					value !== options.sourceFile &&
					value !== options.documentationFile &&
					value.match( jsonFilenameRegex )
				);
			} );

			translatedFiles.forEach( function ( languageFile ) {
				var language = languageFile.match( jsonFilenameRegex )[ 1 ],
					languageMesages = messages( languageFile, language ),
					keys = keysNoMetadata( languageMesages, language ),
					blanks = [],
					duplicates = [],
					unuseds = [],
					missing = sourceMessageKeys.slice( 0 );

				for ( index in keys ) {
					message = keys[ index ];

					if ( missing.indexOf( message ) !== -1 ) {
						if ( languageMesages[ message ] === sourceMessages[ message ] ) {
							duplicates.push( message );
						}
						missing.splice( missing.indexOf( message ), 1 );
					} else {
						unuseds.push( message );
					}

					if ( typeof languageMesages[ message ] !== 'string' ) {
						continue;
					}
					if ( languageMesages[ message ].trim() === '' ) {
						blanks.push( message );
					}
				}

				translatedData[ language ] = {
					messages: languageMesages,
					keys: keys,
					blank: blanks,
					duplicate: duplicates,
					unused: unuseds,
					missing: missing
				};
			} );

			if ( options.requireLowerCase === 'initial' ) {
				sourceMessageWrongCase = sourceMessageKeys.filter( function ( value ) {
					return ( value !== '' && value[ 0 ] !== value[ 0 ].toLowerCase() );
				} );
			} else if ( options.requireLowerCase ) {
				sourceMessageWrongCase = sourceMessageKeys.filter( function ( value ) {
					return value !== value.toLowerCase();
				} );
			}

			if ( options.requireKeyPrefix.length ) {
				if ( typeof options.requireKeyPrefix === 'string' ) {
					options.requireKeyPrefix = [ options.requireKeyPrefix ];
				}
				sourceMessageWrongPrefix = sourceMessageKeys.filter( function ( key ) {
					return options.requireKeyPrefix
						.map( function ( prefix ) {
							return !key.startsWith( prefix );
						} ).reduce( function ( failed, accumulator ) {
							return accumulator && failed;
						} );
				} );
			}

			while ( sourceMessageKeys.length > 0 ) {
				message = sourceMessageKeys[ 0 ];

				offset = documentationMessageKeys.indexOf( message );

				if ( offset !== -1 ) {

					if ( documentationMessages[ message ].trim() === '' ) {
						documentationMessageBlanks.push( message );
					}

					documentationMessageKeys.splice( offset, 1 );
				} else {
					sourceMessageMissing.push( message );
				}
				sourceMessageKeys.splice( 0, 1 );
			}

			if ( options.requireCompleteMessageDocumentation ) {
				// Filter out any missing message that is OK to be skipped
				sourceMessageMissing = sourceMessageMissing.filter( function ( value ) {
					return options.skipIncompleteMessageDocumentation.indexOf( value ) === -1;
				} );
				count = sourceMessageMissing.length;<commit_msg>Simplify filter-map-reduce call to just filter-some<commit_after>/*!
 * A Grunt checker for the 'banana' format JSON i18n message files.
 */

/* eslint-env node */
module.exports = function ( grunt ) {
	grunt.registerMultiTask( 'banana', function () {
		var ok,
			path = require( 'path' ),
			fs = require( 'fs' ),
			options = this.options( {
				sourceFile: 'en.json',
				documentationFile: 'qqq.json',

				requireMetadata: true,

				requireCompleteMessageDocumentation: true,
				disallowUnusedDocumentation: true,

				disallowBlankTranslations: true,
				disallowDuplicateTranslations: false,
				disallowUnusedTranslations: false,

				requireLowerCase: true,
				requireKeyPrefix: [],

				requireCompleteTranslationLanguages: [],
				requireCompleteTranslationMessages: [],

				skipIncompleteMessageDocumentation: []
			} ),
			messageCount = 0;

		if ( this.filesSrc.length === 0 ) {
			grunt.log.error( 'Target directory does not exist.' );
			return false;
		}

		ok = true;

		this.filesSrc.forEach( function ( dir ) {
			var message, index, offset,
				// Source message data
				sourceMessages, sourceMessageKeys,
				// Documentation message data
				documentationMessages, documentationMessageKeys,
				// Translated message data
				translatedFiles,
				jsonFilenameRegex = /(.*)\.json$/,
				translatedData = {},
				documentationMessageBlanks = [],
				sourceMessageMissing = [],
				sourceMessageWrongCase = [],
				sourceMessageWrongPrefix = [],
				count = 0;

			function messages( filename, type ) {
				var messageArray;

				try {
					messageArray = grunt.file.readJSON( path.resolve( dir, filename ) );
				} catch ( e ) {
					grunt.log.error( 'Loading ' + type + ' messages failed: "' + e + '".' );
					ok = false;
					throw e;
				}

				return messageArray;
			}

			function keysNoMetadata( messageArray, type ) {
				var keys, offset;

				try {
					keys = Object.keys( messageArray );
				} catch ( e ) {
					grunt.log.error( 'Loading ' + type + ' messages failed: "' + e + '".' );
					ok = false;
					throw e;
				}

				offset = keys.indexOf( '@metadata' );
				if ( offset === -1 ) {
					if ( options.requireMetadata ) {
						grunt.log.error( 'No metadata block in the ' + type + ' messages file.' );
						ok = false;
					}
				} else {
					keys.splice( offset, 1 );
				}

				return keys;
			}

			sourceMessages = messages( options.sourceFile, 'source' );
			sourceMessageKeys = keysNoMetadata( sourceMessages, 'source' );

			documentationMessages = messages( options.documentationFile, 'documentation' );
			documentationMessageKeys = keysNoMetadata( documentationMessages, 'documentation' );

			// Count after @metadata is removed
			messageCount += sourceMessageKeys.length;

			translatedFiles = fs.readdirSync( dir ).filter( function ( value ) {
				return (
					value !== options.sourceFile &&
					value !== options.documentationFile &&
					value.match( jsonFilenameRegex )
				);
			} );

			translatedFiles.forEach( function ( languageFile ) {
				var language = languageFile.match( jsonFilenameRegex )[ 1 ],
					languageMesages = messages( languageFile, language ),
					keys = keysNoMetadata( languageMesages, language ),
					blanks = [],
					duplicates = [],
					unuseds = [],
					missing = sourceMessageKeys.slice( 0 );

				for ( index in keys ) {
					message = keys[ index ];

					if ( missing.indexOf( message ) !== -1 ) {
						if ( languageMesages[ message ] === sourceMessages[ message ] ) {
							duplicates.push( message );
						}
						missing.splice( missing.indexOf( message ), 1 );
					} else {
						unuseds.push( message );
					}

					if ( typeof languageMesages[ message ] !== 'string' ) {
						continue;
					}
					if ( languageMesages[ message ].trim() === '' ) {
						blanks.push( message );
					}
				}

				translatedData[ language ] = {
					messages: languageMesages,
					keys: keys,
					blank: blanks,
					duplicate: duplicates,
					unused: unuseds,
					missing: missing
				};
			} );

			if ( options.requireLowerCase === 'initial' ) {
				sourceMessageWrongCase = sourceMessageKeys.filter( function ( value ) {
					return ( value !== '' && value[ 0 ] !== value[ 0 ].toLowerCase() );
				} );
			} else if ( options.requireLowerCase ) {
				sourceMessageWrongCase = sourceMessageKeys.filter( function ( value ) {
					return value !== value.toLowerCase();
				} );
			}

			if ( options.requireKeyPrefix.length ) {
				if ( typeof options.requireKeyPrefix === 'string' ) {
					options.requireKeyPrefix = [ options.requireKeyPrefix ];
				}
				sourceMessageWrongPrefix = sourceMessageKeys.filter( function ( key ) {
					return !options.requireKeyPrefix.some( function ( prefix ) {
						return key.startsWith( prefix );
					} );
				} );
			}

			while ( sourceMessageKeys.length > 0 ) {
				message = sourceMessageKeys[ 0 ];

				offset = documentationMessageKeys.indexOf( message );

				if ( offset !== -1 ) {

					if ( documentationMessages[ message ].trim() === '' ) {
						documentationMessageBlanks.push( message );
					}

					documentationMessageKeys.splice( offset, 1 );
				} else {
					sourceMessageMissing.push( message );
				}
				sourceMessageKeys.splice( 0, 1 );
			}

			if ( options.requireCompleteMessageDocumentation ) {
				// Filter out any missing message that is OK to be skipped
				sourceMessageMissing = sourceMessageMissing.filter( function ( value ) {
					return options.skipIncompleteMessageDocumentation.indexOf( value ) === -1;
				} );
				count = sourceMessageMissing.length;
{'fraction_non_alphanumeric': 0.12425929251212067, 'fraction_numerical': 0.003591309032142216, 'mean_word_length': 2.8043032786885247, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7009303', 'n_tokens_mistral': 4254, 'n_tokens_neox': 3132, 'n_words': 758}