<commit_before>const Tokenizer = require( '../src/Tokenizer' ),
	GroupToken = require( '../src/rtf/Group' ),
	GroupEndToken = require( '../src/rtf/GroupEnd' ),
	CommandToken = require( '../src/rtf/Command' ),
	TextToken = require( '../src/rtf/Text' ),
	EscapeToken = require( '../src/rtf/Escape' );

describe( 'Tokenizer integration', () => {
	let mock;

	beforeEach( () => {
		mock = new Tokenizer();
	} );

	describe( 'process', () => {
		it( 'works with simple single-line markup', () => {
			mock.process( '{\\rtf1 foobar}' );

			let ret = mock._results;
			expect( ret.length ).to.be.equal( 4 );
			expect( ret[ 0 ], 'ret[ 0 ]' ).to.be.instanceof( GroupToken );
			expect( ret[ 1 ], 'ret[ 1 ]' ).to.be.instanceof( CommandToken );
			expect( ret[ 1 ].value, 'ret[ 1 ].value' ).to.be.equal( '\\rtf1 ' );
			expect( ret[ 2 ], 'ret[ 2 ]' ).to.be.instanceof( TextToken );
			expect( ret[ 3 ], 'ret[ 3 ]' ).to.be.instanceof( GroupEndToken );
		} );

		it( 'works with multiline markup', () => {
			mock.process( '{\\rtf1 foobar\r\n{abcd}}' );

			let ret = mock._results;
			expect( ret.length ).to.be.equal( 7 );
			expect( ret[ 0 ], 'ret[ 0 ]' ).to.be.instanceof( GroupToken );
			expect( ret[ 1 ], 'ret[ 1 ]' ).to.be.instanceof( CommandToken );
			expect( ret[ 1 ].value, 'ret[ 1 ].value' ).to.be.equal( '\\rtf1 ' );
			expect( ret[ 2 ], 'ret[ 2 ]' ).to.be.instanceof( TextToken );
			expect( ret[ 2 ].value, 'ret[ 2 ] value' ).to.be.equal( 'foobar' );
			expect( ret[ 3 ], 'ret[ 3 ]' ).to.be.instanceof( GroupToken );
			expect( ret[ 4 ], 'ret[ 4 ]' ).to.be.instanceof( TextToken );
			expect( ret[ 4 ].value, 'ret[ 4 ] value' ).to.be.equal( 'abcd' );
			expect( ret[ 5 ], 'ret[ 5 ]' ).to.be.instanceof( GroupEndToken );
			expect( ret[ 6 ], 'ret[ 6 ]' ).to.be.instanceof( GroupEndToken );
		} );

		it('integrates well with escape tokens', () => {
			mock.process( '{\\rtf1 foo\\}bar}' );

			let ret = mock._results;
			expect( ret.length ).to.be.equal( 6 );
			expect( ret[ 0 ], 'ret[ 0 ]' ).to.be.instanceof( GroupToken );
			expect( ret[ 1 ], 'ret[ 1 ]' ).to.be.instanceof( CommandToken );
			expect( ret[ 1 ].value, 'ret[ 1 ].value' ).to.be.equal( '\\rtf1 ' );
			expect( ret[ 2 ], 'ret[ 2 ]' ).to.be.instanceof( TextToken );
			expect( ret[ 2 ].value, 'ret[ 2 ].value' ).to.be.equal( 'foo' );
			expect( ret[ 3 ], 'ret[ 3 ]' ).to.be.instanceof( EscapeToken );
			expect( ret[ 3 ].value, 'ret[ 3 ].value' ).to.be.equal( '\\}' );
			expect( ret[ 4 ], 'ret[ 4 ]' ).to.be.instanceof( TextToken );
			expect( ret[ 4 ].value, 'ret[ 4 ].value' ).to.be.equal( 'bar' );
			expect( ret[ 5 ], 'ret[ 5 ]' ).to.be.instanceof( GroupEndToken );
		});

	} );
} );<commit_msg>Simplified token integration tests with a helper function.<commit_after>const Tokenizer = require( '../src/Tokenizer' ),
	GroupToken = require( '../src/rtf/Group' ),
	GroupEndToken = require( '../src/rtf/GroupEnd' ),
	CommandToken = require( '../src/rtf/Command' ),
	TextToken = require( '../src/rtf/Text' ),
	EscapeToken = require( '../src/rtf/Escape' );

describe( 'Tokenizer integration', () => {
	let mock;

	beforeEach( () => {
		mock = new Tokenizer();
	} );

	/**
	 * Helper function to assert tokens returned by Tokenizer.
	 *
	 * @param {Array[]/Function[]} expected Either array of expected types, or array of arrays in format `[ <type>, <expectedStringValue> ]`.
	 * @param {Token[]} actual
	 */
	function assertParsedTokens( expected, actual ) {
		if ( !Array.isArray( expected ) ) {
			throw new TypeError( 'assertParsedTokens expects to get an array' );
		}

		expect( actual.length, 'ret.length' ).to.be.equal( expected.length, 'expected.length' );

		for ( let i = 0; i < expected.length; i++ ) {
			let curExpect = expected[ i ];

			if ( typeof curExpect === 'function' ) {
				// Simply checking type.
				expect( actual[ i ], `ret[${i}]` ).to.be.instanceof( curExpect );
			} else {
				// Array format.
				expect( actual[ i ], `ret[${i}]` ).to.be.instanceof( curExpect[ 0 ] );
				expect( actual[ i ].value, `ret[${i}].value` ).to.be.equal( curExpect[ 1 ] );
			}
		}
	}

	describe( 'process', () => {
		it( 'works with simple single-line markup', () => {
			mock.process( '{\\rtf1 foobar}' );

			let ret = mock._results;

			assertParsedTokens( [
				GroupToken, [ CommandToken, '\\rtf1 ' ],
				[ TextToken, 'foobar' ],
				GroupEndToken
			], ret );
		} );

		it( 'works with multiline markup', () => {
			mock.process( '{\\rtf1 foobar\r\n{abcd}}' );

			let ret = mock._results;

			assertParsedTokens( [
				GroupToken, [ CommandToken, '\\rtf1 ' ],
				[ TextToken, 'foobar' ],
				GroupToken, [ TextToken, 'abcd' ],
				GroupEndToken,
				GroupEndToken
			], ret );
		} );

		it( 'integrates well with escape tokens', () => {
			mock.process( '{\\rtf1 foo\\}bar}' );

			let ret = mock._results;

			assertParsedTokens( [
				GroupToken, [ CommandToken, '\\rtf1 ' ],
				[ TextToken, 'foo' ],
				[ EscapeToken, '\\}' ],
				[ TextToken, 'bar' ],
				GroupEndToken
			], ret );
		} );

	} );
} );
{'fraction_non_alphanumeric': 0.2138138138138138, 'fraction_numerical': 0.013613613613613613, 'mean_word_length': 2.9682287529785545, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3292262', 'n_tokens_mistral': 2088, 'n_tokens_neox': 1815, 'n_words': 460}