<commit_before>package co.m4solutions.grammars.service;

import co.m4solutions.grammars.model.Grammar;
import co.m4solutions.grammars.model.KeyWord;
import co.m4solutions.grammars.model.NonTerminalToken;
import co.m4solutions.grammars.model.TerminalToken;
import co.m4solutions.grammars.model.Token;

public class TokenFactory {

	public static Token buildToken(String tk) {
		tk = tk.trim();
		if(tk.contains("<")){
			tk = tk.replaceAll("<", "").replaceAll(">", "").trim();
			Token tokenObj = Grammar.getTokenHolder().getToken(tk);
			if(tokenObj!=null){
				return tokenObj;
			}else{
				tokenObj = new NonTerminalToken(tk);
				Grammar.getTokenHolder().addToken(tokenObj);
				return tokenObj;
			}
			
		}
		
		if(tk.contains("$")){
			tk = tk.replaceAll("\\$", "").trim();
			
			Token tokenObj = Grammar.getTokenHolder().getToken(tk);
			if(tokenObj!=null){
				return tokenObj;
			}else{
				tokenObj = new KeyWord(tk);
				Grammar.getTokenHolder().addToken(tokenObj);
				return tokenObj;
			}
		}
		<commit_msg>Update TokenFactory.java<commit_after>package co.m4solutions.grammars.service;

import co.m4solutions.grammars.model.Grammar;
import co.m4solutions.grammars.model.KeyWord;
import co.m4solutions.grammars.model.NonTerminalToken;
import co.m4solutions.grammars.model.TerminalToken;
import co.m4solutions.grammars.model.Token;

public class TokenFactory {
	//TODO: refactor this method in it's coresponding sub-methods or classes.
	public static Token buildToken(String tk) {
		tk = tk.trim();
		if(tk.contains("<")){
			tk = tk.replaceAll("<", "").replaceAll(">", "").trim();
			Token tokenObj = Grammar.getTokenHolder().getToken(tk);
			if(tokenObj!=null){
				return tokenObj;
			}else{
				tokenObj = new NonTerminalToken(tk);
				Grammar.getTokenHolder().addToken(tokenObj);
				return tokenObj;
			}
			
		}
		
		if(tk.contains("$")){
			tk = tk.replaceAll("\\$", "").trim();
			
			Token tokenObj = Grammar.getTokenHolder().getToken(tk);
			if(tokenObj!=null){
				return tokenObj;
			}else{
				tokenObj = new KeyWord(tk);
				Grammar.getTokenHolder().addToken(tokenObj);
				return tokenObj;
			}
		}
		
{'fraction_non_alphanumeric': 0.15157203190990146, 'fraction_numerical': 0.005631159080244017, 'mean_word_length': 4.112709832134293, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6037689', 'n_tokens_mistral': 875, 'n_tokens_neox': 754, 'n_words': 125}