<commit_before>/*
 * <NAME> Feb 19, 2010
 */
package net.colar.netbeans.fan.parboiled;

import java.util.Hashtable;
import net.colar.netbeans.fan.FanTokenID;
import net.colar.netbeans.fan.FanUtilities;

/**
 * Parboiled parser does not really provide a lexer, so we simulate one using the parser.
 * This creates the tokens important to the lexer (syntax highlighting)
 * @author thibautc
 */
public class FantomLexerTokens
{
	// Lexer label names ... we want all tokens provided by the lexer() rule of the parser to be listed here
	public enum TokenName{
		COMMENT, UNIXLINE, DOC, LEXEROPS, LEXERSEPS, LEXERASSIGN, LEXERINIT, LEXERCOMPS, LEXERITEMS,
		STRS, URI, CHAR_, KEYWORD, ID, LEXERID, NUMBER, WHITESPACE, SPACING, LF, DSL, ANY, ERROR, UNEXPECTED,
		//Items we use a lot in the lexer
		SQ_BRACKET_L, SQ_BRACKET_R, BRACKET_L, BRACKET_R, PAR_L, PAR_R, DOT, AT, DSL_OPEN, DSL_CLOSE
	}

	// lazy inited
	private static Hashtable<Integer, FanTokenID> tokens = new Hashtable<Integer, FanTokenID>();
	private static Hashtable<String, Integer> tokenIdByName = new Hashtable<String, Integer>();

	// Map Token lables with Netbeans color entries (textColors.xml)
	private static Hashtable<TokenName, String> getColorIds()
	{
		Hashtable<TokenName, String> cats = new Hashtable<TokenName, String>();
		cats.put(TokenName.COMMENT, "comment");
		cats.put(TokenName.DOC, "DOC");
		cats.put(TokenName.UNIXLINE, "DOC");
		cats.put(TokenName.DSL, "DSL");
		cats.put(TokenName.NUMBER, "number");
		cats.put(TokenName.CHAR_, "character");
		cats.put(TokenName.KEYWORD, "keyword");
		cats.put(TokenName.LEXEROPS, "operator");
		cats.put(TokenName.LEXERSEPS, "separator");
		cats.put(TokenName.LEXERCOMPS, "comparator");
		cats.put(TokenName.LEXERASSIGN, "assignment");
		cats.put(TokenName.LEXERASSIGN, "initialization");
		cats.put(TokenName.STRS, "string");
		cats.put(TokenName.URI, "string");
		cats.put(TokenName.ERROR, "error"); //error token in case of problem with lexer
		cats.put(TokenName.UNEXPECTED, "error"); //should NOT happen
		return cats;
	}

	/**
	 * Get a token object by it's label/name
	 * @param name
	 * @return
	 */
	public static FanTokenID getTokenByName(String name)
	{
		if(name==null)
		{
			FanUtilities.GENERIC_LOGGER.info("FantomparserToken: Null token passed");
			return getTokenByName(TokenName.ERROR);
		}
		Integer id = tokenIdByName.get(name.toUpperCase());
		if(id==null)
		{
			FanUtilities.GENERIC_LOGGER.info("FantomparserToken: No such token: "+name);
			return getTokenByName(TokenName.ERROR);
		}
		return tokens.get(id);
	}

	public static FanTokenID getTokenByName(TokenName name)
	{
		return getTokenByName(name.name().toUpperCase());
	}

	/**
	 * Get all the defined token (lazilly cached)
	 * @return
	 */
	public static Hashtable<Integer, FanTokenID> getTokens()
	{
		if (tokens.isEmpty())
		{
			synchronized(FantomParser.class)
			{
				if (tokens.isEmpty())
				{
					tokens = new Hashtable<Integer, FanTokenID>();
					Hashtable<TokenName, String> colorIds = getColorIds();

					int id = 0;
					for(TokenName name : TokenName.values())
					{
						id++;
						FanTokenID token;
						String cat = "";
						if (colorIds.containsKey(name))
						{
							cat = (String) colorIds.get(name);
						}
						//System.out.println("New token: " + name + "(" + id + ") : " + cat);
						token = new FanTokenID(name.name(), id, cat);
						tokens.put(id, token);
						tokenIdByName.put(name.name(), id);
					}
				}
			}
		}
		return tokens;
	}
}
<commit_msg>SuppressWarning<commit_after>/*
 * <NAME> Feb 19, 2010
 */
package net.colar.netbeans.fan.parboiled;

import java.util.Hashtable;
import net.colar.netbeans.fan.FanTokenID;
import net.colar.netbeans.fan.FanUtilities;

/**
 * Parboiled parser does not really provide a lexer, so we simulate one using the parser.
 * This creates the tokens important to the lexer (syntax highlighting)
 * @author thibautc
 */
public class FantomLexerTokens
{
	// Lexer label names ... we want all tokens provided by the lexer() rule of the parser to be listed here
	public enum TokenName{
		COMMENT, UNIXLINE, DOC, LEXEROPS, LEXERSEPS, LEXERASSIGN, LEXERINIT, LEXERCOMPS, LEXERITEMS,
		STRS, URI, CHAR_, KEYWORD, ID, LEXERID, NUMBER, WHITESPACE, SPACING, LF, DSL, ANY, ERROR, UNEXPECTED,
		//Items we use a lot in the lexer
		SQ_BRACKET_L, SQ_BRACKET_R, BRACKET_L, BRACKET_R, PAR_L, PAR_R, DOT, AT, DSL_OPEN, DSL_CLOSE
	}

	// lazy inited
	private static Hashtable<Integer, FanTokenID> tokens = new Hashtable<Integer, FanTokenID>();
	private static Hashtable<String, Integer> tokenIdByName = new Hashtable<String, Integer>();

	// Map Token lables with Netbeans color entries (textColors.xml)
	private static Hashtable<TokenName, String> getColorIds()
	{
		Hashtable<TokenName, String> cats = new Hashtable<TokenName, String>();
		cats.put(TokenName.COMMENT, "comment");
		cats.put(TokenName.DOC, "DOC");
		cats.put(TokenName.UNIXLINE, "DOC");
		cats.put(TokenName.DSL, "DSL");
		cats.put(TokenName.NUMBER, "number");
		cats.put(TokenName.CHAR_, "character");
		cats.put(TokenName.KEYWORD, "keyword");
		cats.put(TokenName.LEXEROPS, "operator");
		cats.put(TokenName.LEXERSEPS, "separator");
		cats.put(TokenName.LEXERCOMPS, "comparator");
		cats.put(TokenName.LEXERASSIGN, "assignment");
		cats.put(TokenName.LEXERASSIGN, "initialization");
		cats.put(TokenName.STRS, "string");
		cats.put(TokenName.URI, "string");
		cats.put(TokenName.ERROR, "error"); //error token in case of problem with lexer
		cats.put(TokenName.UNEXPECTED, "error"); //should NOT happen
		return cats;
	}

	/**
	 * Get a token object by it's label/name
	 * @param name
	 * @return
	 */
	public static FanTokenID getTokenByName(String name)
	{
		if(name==null)
		{
			FanUtilities.GENERIC_LOGGER.info("FantomparserToken: Null token passed");
			return getTokenByName(TokenName.ERROR);
		}
		Integer id = tokenIdByName.get(name.toUpperCase());
		if(id==null)
		{
			FanUtilities.GENERIC_LOGGER.info("FantomparserToken: No such token: "+name);
			return getTokenByName(TokenName.ERROR);
		}
		return tokens.get(id);
	}

	public static FanTokenID getTokenByName(TokenName name)
	{
		return getTokenByName(name.name().toUpperCase());
	}

	/**
	 * Get all the defined token (lazilly cached)
	 * @return
	 */
	public static Hashtable<Integer, FanTokenID> getTokens()
	{
		if (tokens.isEmpty())
		{
			synchronized(FantomParser.class)
			{
				if (tokens.isEmpty())
				{
					tokens = new Hashtable<Integer, FanTokenID>();
					Hashtable<TokenName, String> colorIds = getColorIds();

					int id = 0;
					for(TokenName name : TokenName.values())
					{
						id++;
						FanTokenID token;
						String cat = "";
						if (colorIds.containsKey(name))
						{
							cat = colorIds.get(name);
						}
						//System.out.println("New token: " + name + "(" + id + ") : " + cat);
						token = new FanTokenID(name.name(), id, cat);
						tokens.put(id, token);
						tokenIdByName.put(name.name(), id);
					}
				}
			}
		}
		return tokens;
	}
}

{'fraction_non_alphanumeric': 0.13417250750965526, 'fraction_numerical': 0.0020025747389500787, 'mean_word_length': 3.9871611982881596, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '529843', 'n_tokens_mistral': 2835, 'n_tokens_neox': 2433, 'n_words': 575}