<commit_before>package org.eclipse.xtext.lexer;

import org.eclipse.emf.common.util.EList;
import org.eclipse.emf.ecore.EObject;
import org.eclipse.xtext.AbstractRule;
import org.eclipse.xtext.BuiltinRules;
import org.eclipse.xtext.GrammarUtil;
import org.eclipse.xtext.RuleCall;
import org.eclipse.xtext.core.parser.ITokenTypes;
import org.eclipse.xtext.core.parsetree.AbstractNode;
import org.eclipse.xtext.core.parsetree.LeafNode;
import org.eclipse.xtext.core.parsetree.NodeAdapter;
import org.eclipse.xtext.generator.tests.AbstractGeneratorTest;
import org.eclipse.xtext.lexer.parser.LexerLanguageASTFactory;

public class LexerTokenTest extends AbstractGeneratorTest {

	public void testLexerTokens() throws Exception {
		AbstractNode rootNode = getRootNode("xyz A CCC BB");
		EList<LeafNode> leafNodes = rootNode.getLeafNodes();
		assertTrue(GrammarUtil.isLexerRuleCall(leafNodes.get(0).getGrammarElement()));
		assertEquals(BuiltinRules.BUILTIN_ID, GrammarUtil.calledRule((RuleCall) leafNodes.get(0).getGrammarElement()));
		assertEquals(ITokenTypes.IDENTIFIER, leafNodes.get(0).tokenType());
		checkIsWhitespace(leafNodes.get(1));
		checkIsDefinedRule(leafNodes.get(2), "EXPLICITTOKENTYPE", LexerLanguageTokenTypes.X);
		checkIsWhitespace(leafNodes.get(3));
		checkIsDefinedRule(leafNodes.get(4), "IMPLICITTOKENTYPE", LexerLanguageTokenTypes.IMPLICITTOKENTYPE);
		checkIsWhitespace(leafNodes.get(5));
		checkIsDefinedRule(leafNodes.get(6), "STRING", LexerLanguageTokenTypes.STRING);
		
	}
	
	private void checkIsDefinedRule(LeafNode leafNode, String ruleName, String tokenType) {
		assertTrue(GrammarUtil.isLexerRuleCall(leafNode.getGrammarElement()));
		AbstractRule calledRule = GrammarUtil.calledRule((RuleCall) leafNode.getGrammarElement());
		assertFalse(BuiltinRules.getBuiltinLexerRules().contains(calledRule));
		assertEquals(ruleName, calledRule.getName());
		assertEquals(tokenType, leafNode.tokenType());
	}
	
	private void checkIsWhitespace(LeafNode leafNode) {
		assertEquals(BuiltinRules.BUILTIN_WS, leafNode.getGrammarElement());
		assertEquals(ITokenTypes.WHITESPACE, leafNode.tokenType());
	}
	
	private EObject getRootAstElement(String model) throws Exception {
		EObject object = (EObject) parse(model, new LexerLanguageASTFactory());
		return object;
	}
	
	private AbstractNode getRootNode(String model) throws Exception {
		EObject object = getRootAstElement(model);
		NodeAdapter adapter = (NodeAdapter) object.eAdapters().get(0);
		AbstractNode node = adapter.getParserNode();
		return node;
	}

	@Override
	protected Class<?> getTheClass() {
		return LexerLanguage.class;
	}
}<commit_msg>default values for tokenTypes<commit_after>package org.eclipse.xtext.lexer;

import org.eclipse.emf.common.util.EList;
import org.eclipse.xtext.AbstractRule;
import org.eclipse.xtext.BuiltinRules;
import org.eclipse.xtext.GrammarUtil;
import org.eclipse.xtext.RuleCall;
import org.eclipse.xtext.core.parser.IElementFactory;
import org.eclipse.xtext.core.parser.ITokenTypes;
import org.eclipse.xtext.core.parsetree.AbstractNode;
import org.eclipse.xtext.core.parsetree.LeafNode;
import org.eclipse.xtext.generator.tests.AbstractGeneratorTest;
import org.eclipse.xtext.lexer.parser.LexerLanguageASTFactory;

public class LexerTokenTest extends AbstractGeneratorTest {

	public void testLexerTokens() throws Exception {
		AbstractNode rootNode = getRootNode("xyz A CCC BB");
		EList<LeafNode> leafNodes = rootNode.getLeafNodes();
		assertTrue(GrammarUtil.isLexerRuleCall(leafNodes.get(0).getGrammarElement()));
		assertEquals(BuiltinRules.BUILTIN_ID, GrammarUtil.calledRule((RuleCall) leafNodes.get(0).getGrammarElement()));
		assertEquals(ITokenTypes.IDENTIFIER, leafNodes.get(0).tokenType());
		checkIsWhitespace(leafNodes.get(1));
		checkIsDefinedRule(leafNodes.get(2), "EXPLICITTOKENTYPE", LexerLanguageTokenTypes.X);
		checkIsWhitespace(leafNodes.get(3));
		checkIsDefinedRule(leafNodes.get(4), "IMPLICITTOKENTYPE", LexerLanguageTokenTypes.IMPLICITTOKENTYPE);
		checkIsWhitespace(leafNodes.get(5));
		checkIsDefinedRule(leafNodes.get(6), "STRING", LexerLanguageTokenTypes.STRING);
		
	}
	
	private void checkIsDefinedRule(LeafNode leafNode, String ruleName, String tokenType) {
		assertTrue(GrammarUtil.isLexerRuleCall(leafNode.getGrammarElement()));
		AbstractRule calledRule = GrammarUtil.calledRule((RuleCall) leafNode.getGrammarElement());
		assertFalse(BuiltinRules.getBuiltinLexerRules().contains(calledRule));
		assertEquals(ruleName, calledRule.getName());
		assertEquals(tokenType, leafNode.tokenType());
	}
	
	private void checkIsWhitespace(LeafNode leafNode) {
		assertEquals(BuiltinRules.BUILTIN_WS, leafNode.getGrammarElement());
		assertEquals(ITokenTypes.WHITESPACE, leafNode.tokenType());
	}
	
	@Override
	protected Class<?> getTheClass() {
		return LexerLanguage.class;
	}
	
	@Override
	protected IElementFactory getASTFactory() throws Exception {
		return new LexerLanguageASTFactory();
	}
}
{'fraction_non_alphanumeric': 0.11253041362530414, 'fraction_numerical': 0.0038523925385239256, 'mean_word_length': 7.446917808219178, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3877214', 'n_tokens_mistral': 1661, 'n_tokens_neox': 1584, 'n_words': 232}