<commit_before>( function() {
	'use strict';

	const Text = require( './rtf/Text' ),
		EventEmitter = require( 'events' );

	class Tokenizer extends EventEmitter {

		constructor() {
			super();
			// this.splitRegExp = new RegExp( Tokenizer.RTF_NEW_LINE, 'm' );
			this.splitRegExp = /\r\n/m;
			this._loadTokens();

			this._results = [];

			this.on( 'matched', token => this._results.push( token ) );
		}

		process( code ) {
			code = String( code );

			// Tokenizer splits RTF content into chunks per space / new line.
			let separatorMatch = this.splitRegExp.exec( code ),
				chunk = separatorMatch ? code.substr( 0, separatorMatch.index ) : code,
				remaining = separatorMatch ? code.substr( separatorMatch.index + separatorMatch[ 0 ].length ) : null;

			while ( chunk ) {
				let closestMatch = null,
					matchedText = null,
					// At what offset the token was matched?
					matchIndex = null;

				for ( let token of this.tokens ) {
					let tokenMatched = token.match( chunk );

					// Only overwrite closest match if:
					// 1. No token has been matched so far.
					// 2. ... or this token has been found closer to the end of a string than the former one.
					if ( tokenMatched && ( matchIndex === null || tokenMatched[ 0 ] < matchIndex ) ) {
						closestMatch = token;
						matchIndex = tokenMatched[ 0 ];
						matchedText = tokenMatched[ 1 ];
					}

					if ( matchIndex === 0 ) {
						// Any token matched at 0 index, takes highest priority, and means no further processing is needed.
						break;
					}
				}

				if ( matchIndex !== 0 ) {
					// console.log( `not having a perfect match (${matchIndex}) :(( using text` );

					// No match perfect match, so we'll create a text run all the way until a best match. The best
					// match will be picked in next iteration at offset 0.
					matchedText = matchIndex !== null ? chunk.substring( 0, matchIndex ) : chunk;
					closestMatch = new Text( matchedText );
					matchIndex = 0;
				} else {
					// @todo: until there's no difference between token and model, we need to use the following.
					closestMatch = new closestMatch.constructor( matchedText );
				}

				// Update current chunk.
				chunk = chunk.substr( matchedText.length );

				this.emit( 'matched', closestMatch );
			}

			if ( remaining && remaining.length ) {
				this.process( remaining );
			}
		}

		/**
		 * New line feed used in RTF files.
		 *
		 * @readonly
		 * @static
		 * @memberOf Tokenizer
		 */
		static get RTF_NEW_LINE() {
			return '\r\n';
		}

		/**
		 * Sets up known tokens array.
		 *
		 * @memberOf Tokenizer
		 */
		_loadTokens() {
			const Group = require( './rtf/Group' ),
				GroupEnd = require( './rtf/GroupEnd' ),
				Command = require( './rtf/Command' ),
				Escape = require( './rtf/Escape' );

			this.tokens = [
				new Group(),
				new GroupEnd(),
				new Command()
			];

			// It's important to put Escape token at the end as it's the last one that should be used.
			this.tokens.push( new Escape() );

			this.tokens[ 0 ].name = 'Group';
			this.tokens[ 1 ].name = 'GroupEnd';
		}
	}<commit_msg>Removed leftovers.<commit_after>( function() {
	'use strict';

	const Text = require( './rtf/Text' ),
		EventEmitter = require( 'events' );

	class Tokenizer extends EventEmitter {

		constructor() {
			super();
			// this.splitRegExp = new RegExp( Tokenizer.RTF_NEW_LINE, 'm' );
			this.splitRegExp = /\r\n/m;
			this._loadTokens();

			this._results = [];

			this.on( 'matched', token => this._results.push( token ) );
		}

		process( code ) {
			code = String( code );

			// Tokenizer splits RTF content into chunks per space / new line.
			let separatorMatch = this.splitRegExp.exec( code ),
				chunk = separatorMatch ? code.substr( 0, separatorMatch.index ) : code,
				remaining = separatorMatch ? code.substr( separatorMatch.index + separatorMatch[ 0 ].length ) : null;

			while ( chunk ) {
				let closestMatch = null,
					matchedText = null,
					// At what offset the token was matched?
					matchIndex = null;

				for ( let token of this.tokens ) {
					let tokenMatched = token.match( chunk );

					// Only overwrite closest match if:
					// 1. No token has been matched so far.
					// 2. ... or this token has been found closer to the end of a string than the former one.
					if ( tokenMatched && ( matchIndex === null || tokenMatched[ 0 ] < matchIndex ) ) {
						closestMatch = token;
						matchIndex = tokenMatched[ 0 ];
						matchedText = tokenMatched[ 1 ];
					}

					if ( matchIndex === 0 ) {
						// Any token matched at 0 index, takes highest priority, and means no further processing is needed.
						break;
					}
				}

				if ( matchIndex !== 0 ) {
					// console.log( `not having a perfect match (${matchIndex}) :(( using text` );

					// No match perfect match, so we'll create a text run all the way until a best match. The best
					// match will be picked in next iteration at offset 0.
					matchedText = matchIndex !== null ? chunk.substring( 0, matchIndex ) : chunk;
					closestMatch = new Text( matchedText );
					matchIndex = 0;
				} else {
					// @todo: until there's no difference between token and model, we need to use the following.
					closestMatch = new closestMatch.constructor( matchedText );
				}

				// Update current chunk.
				chunk = chunk.substr( matchedText.length );

				this.emit( 'matched', closestMatch );
			}

			if ( remaining && remaining.length ) {
				this.process( remaining );
			}
		}

		/**
		 * New line feed used in RTF files.
		 *
		 * @readonly
		 * @static
		 * @memberOf Tokenizer
		 */
		static get RTF_NEW_LINE() {
			return '\r\n';
		}

		/**
		 * Sets up known tokens array.
		 *
		 * @memberOf Tokenizer
		 */
		_loadTokens() {
			const Group = require( './rtf/Group' ),
				GroupEnd = require( './rtf/GroupEnd' ),
				Command = require( './rtf/Command' ),
				Escape = require( './rtf/Escape' );

			this.tokens = [
				new Group(),
				new GroupEnd(),
				new Command()
			];

			// It's important to put Escape token at the end as it's the last one that should be used.
			this.tokens.push( new Escape() );
		}
	}
{'fraction_non_alphanumeric': 0.12596232596232596, 'fraction_numerical': 0.004586404586404586, 'mean_word_length': 2.4575311438278598, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1826178', 'n_tokens_mistral': 2346, 'n_tokens_neox': 1801, 'n_words': 625}