<commit_before>package org.anarres.cpp;

import java.io.StringReader;

import java.util.Arrays;
import java.util.Iterator;
import java.util.List;

import junit.framework.Test;

import static org.anarres.cpp.Token.*;

public class LexerSourceTestCase extends BaseTestCase implements Test {

	private void testLexerSource(String in, int... out)
						throws Exception {
		System.out.println("Testing '" + in + "' => " +
						Arrays.toString(out));
		StringLexerSource	s = new StringLexerSource(in);

		for (int i = 0; i < out.length; i++) {
			Token	tok = s.token();
			System.out.println("Token is " + tok);
			assertEquals(out[i], tok.getType());
		}
		assertEquals(EOF, s.token().getType());
	}

	public void testLexerSource()
						throws Exception {

		testLexerSource("int a = 5;",
			IDENTIFIER, WHITESPACE, IDENTIFIER, WHITESPACE,
			'=', WHITESPACE, INTEGER, ';', EOF
		);

		// \n is WHITESPACE because ppvalid = false
		testLexerSource("# #   \r\n\n\r \rfoo",
			HASH, WHITESPACE, '#', WHITESPACE, IDENTIFIER
		);

		testLexerSource("%:%:", PASTE);
		testLexerSource("%:?", '#', '?');
		testLexerSource("%:%=", '#', MOD_EQ);
		testLexerSource("0x1234ffdUL 0765I",
				INTEGER, WHITESPACE, INTEGER);

		testLexerSource("+= -= *= /= %= <= >= >>= <<= &= |= ^= x",
			PLUS_EQ, WHITESPACE,
			SUB_EQ, WHITESPACE,
			MULT_EQ, WHITESPACE,
			DIV_EQ, WHITESPACE,
			MOD_EQ, WHITESPACE,
			LE, WHITESPACE,
			GE, WHITESPACE,
			RSH_EQ, WHITESPACE,
			LSH_EQ, WHITESPACE,
			AND_EQ, WHITESPACE,
			OR_EQ, WHITESPACE,
			XOR_EQ, WHITESPACE,
			IDENTIFIER);

		testLexerSource("/**/", CCOMMENT);
		testLexerSource("/** ** **/", CCOMMENT);
		testLexerSource("//* ** **/", CPPCOMMENT);
		testLexerSource("'\\r' '\\xf' '\\xff' 'x' 'aa' ''",
			CHARACTER, WHITESPACE,
			CHARACTER, WHITESPACE,
			CHARACTER, WHITESPACE,
			CHARACTER, WHITESPACE,
			INVALID, WHITESPACE,
			INVALID);

		testLexerSource("1i1I1l1L1ui1ul", 
			INTEGER, INTEGER,
			INTEGER, INTEGER,
			INTEGER, INTEGER);

	}

}
<commit_msg>Add nested-comment test case<commit_after>package org.anarres.cpp;

import java.io.StringReader;

import java.util.Arrays;
import java.util.Iterator;
import java.util.List;

import junit.framework.Test;

import static org.anarres.cpp.Token.*;

public class LexerSourceTestCase extends BaseTestCase implements Test {

	private void testLexerSource(String in, int... out)
						throws Exception {
		System.out.println("Testing '" + in + "' => " +
						Arrays.toString(out));
		StringLexerSource	s = new StringLexerSource(in);

		for (int i = 0; i < out.length; i++) {
			Token	tok = s.token();
			System.out.println("Token is " + tok);
			assertEquals(out[i], tok.getType());
		}
		assertEquals(EOF, s.token().getType());
	}

	public void testLexerSource()
						throws Exception {

		testLexerSource("int a = 5;",
			IDENTIFIER, WHITESPACE, IDENTIFIER, WHITESPACE,
			'=', WHITESPACE, INTEGER, ';', EOF
		);

		// \n is WHITESPACE because ppvalid = false
		testLexerSource("# #   \r\n\n\r \rfoo",
			HASH, WHITESPACE, '#', WHITESPACE, IDENTIFIER
		);

		testLexerSource("%:%:", PASTE);
		testLexerSource("%:?", '#', '?');
		testLexerSource("%:%=", '#', MOD_EQ);
		testLexerSource("0x1234ffdUL 0765I",
				INTEGER, WHITESPACE, INTEGER);

		testLexerSource("+= -= *= /= %= <= >= >>= <<= &= |= ^= x",
			PLUS_EQ, WHITESPACE,
			SUB_EQ, WHITESPACE,
			MULT_EQ, WHITESPACE,
			DIV_EQ, WHITESPACE,
			MOD_EQ, WHITESPACE,
			LE, WHITESPACE,
			GE, WHITESPACE,
			RSH_EQ, WHITESPACE,
			LSH_EQ, WHITESPACE,
			AND_EQ, WHITESPACE,
			OR_EQ, WHITESPACE,
			XOR_EQ, WHITESPACE,
			IDENTIFIER);

		testLexerSource("/**/", CCOMMENT);
		testLexerSource("/* /**/ */", CCOMMENT, WHITESPACE, '*', '/');
		testLexerSource("/** ** **/", CCOMMENT);
		testLexerSource("//* ** **/", CPPCOMMENT);
		testLexerSource("'\\r' '\\xf' '\\xff' 'x' 'aa' ''",
			CHARACTER, WHITESPACE,
			CHARACTER, WHITESPACE,
			CHARACTER, WHITESPACE,
			CHARACTER, WHITESPACE,
			INVALID, WHITESPACE,
			INVALID);

		testLexerSource("1i1I1l1L1ui1ul", 
			INTEGER, INTEGER,
			INTEGER, INTEGER,
			INTEGER, INTEGER);

	}

}

{'fraction_non_alphanumeric': 0.17271835132482827, 'fraction_numerical': 0.008341511285574092, 'mean_word_length': 3.768421052631579, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5389961', 'n_tokens_mistral': 1905, 'n_tokens_neox': 1574, 'n_words': 302}