<commit_before>package edu.asu.plp;

import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Pattern;

import edu.asu.plp.compile.lex.LexException;

public class Token
{
	public static interface Groups
	{
		public static final String[] CONTROL_TOKENS = new String[] { "\\.", "\\(", "\\)",
				"\\{", "}", "\\[", "]", ";", "," };
	}
	
	public static enum Type
	{
		CONTROL(Groups.CONTROL_TOKENS),
		LITERAL_INT("(0(x|b|d))?(\\d)+"),
		LITERAL_LONG("(\\d)+[lL]"),
		LITERAL_FLOAT("(((\\d+)?(\\.\\d+))|((\\d+)(\\.\\d+)?))[fF]"),
		LITERAL_DOUBLE("(\\d+)?(\\.\\d+)"),
		LITERAL_CHAR("'[a-z A-Z]?'"),
		LITERAL_STRING("\"([^\"\\\\\\n\\r]|\\\\.)*\""),
		LITERAL_BOOLEAN("true|false"),
		LITERAL_NULL("null"),
		CONSTRUCT_GENERICS_DEF("<[a-z A-Z]+>"),
		OPERATOR_UNARY("\\+\\+|--"),
		OPERATOR_BINARY("((\\+|<<|>>|-|\\/|\\*|\\||&)=?)|="),
		OPERATOR(OPERATOR_UNARY, OPERATOR_BINARY), // Backward Compatibility
		COMPARATOR(">|>=|<|<=|&&|\\|\\||==|instanceof"),
		TYPE("boolean|long|int|byte|short|char|double|float|void"),
		MODIFIER_ACCESS_PERMISSIONS("public|private|protected"),
		MODIFIER_ACCESS(MODIFIER_ACCESS_PERMISSIONS, "static"), // Backward Compatibility
		MODIFIER_BEHAVIOUR("final|volitile|transient|synchronized|native|abstract|throws"),
		MODIFIER_INHERITENCE("extends|implements"),
		ACTION("return|continue|break|throw|new|assert|strictfp"),
		CONSTRUCT_BLOCK("if|else|do|while|switch|case:|default|for|try|catch|finally"),
		CONSTRUCT_TYPE_DEF("class|interface|enum"),
		SPECIAL_ORGANIZATION("package"),
		SPECIAL_RESERVED("goto|const"),
		SPECIAL_IMPORT("import"),
		REFERNCE("[a-zA-Z]+[a-zA-Z\\d]*"),
		UNSUPPORTED(LITERAL_LONG, LITERAL_FLOAT, LITERAL_DOUBLE, LITERAL_CHAR,
				LITERAL_STRING, "\\/", MODIFIER_INHERITENCE, SPECIAL_RESERVED,
				"try|catch|finally|enum|interface|assert|new|throw", SPECIAL_IMPORT,
				CONSTRUCT_GENERICS_DEF,
				"instanceof|double|float|volitile|transient|synchronized|native|abstract|throws|:");
		
		public String regex;
		
		private Type(String regex)
		{
			this.regex = regex;
		}
		
		private Type(Object first, Object... objects)
		{
			this.regex = compoundRegex(first, objects);
		}
		
		private Type(String[] strings)
		{
			this(strings[0], (Object[]) Arrays.copyOfRange(strings, 1, strings.length));
		}
		
		public boolean matches(String token)
		{<commit_msg>Tokenize the import path with the import statement<commit_after>package edu.asu.plp;

import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Pattern;

import edu.asu.plp.compile.lex.LexException;

public class Token
{
	public static interface Groups
	{
		public static final String[] CONTROL_TOKENS = new String[] { "\\.", "\\(", "\\)",
				"\\{", "}", "\\[", "]", ";", "," };
	}
	
	public static enum Type
	{
		CONTROL(Groups.CONTROL_TOKENS),
		LITERAL_INT("(0(x|b|d))?(\\d)+"),
		LITERAL_LONG("(\\d)+[lL]"),
		LITERAL_FLOAT("(((\\d+)?(\\.\\d+))|((\\d+)(\\.\\d+)?))[fF]"),
		LITERAL_DOUBLE("(\\d+)?(\\.\\d+)"),
		LITERAL_CHAR("'[a-z A-Z]?'"),
		LITERAL_STRING("\"([^\"\\\\\\n\\r]|\\\\.)*\""),
		LITERAL_BOOLEAN("true|false"),
		LITERAL_NULL("null"),
		CONSTRUCT_GENERICS_DEF("<[a-z A-Z]+>"),
		OPERATOR_UNARY("\\+\\+|--"),
		OPERATOR_BINARY("((\\+|<<|>>|-|\\/|\\*|\\||&)=?)|="),
		OPERATOR(OPERATOR_UNARY, OPERATOR_BINARY), // Backward Compatibility
		COMPARATOR(">|>=|<|<=|&&|\\|\\||==|instanceof"),
		TYPE("boolean|long|int|byte|short|char|double|float|void"),
		MODIFIER_ACCESS_PERMISSIONS("public|private|protected"),
		MODIFIER_ACCESS(MODIFIER_ACCESS_PERMISSIONS, "static"), // Backward Compatibility
		MODIFIER_BEHAVIOUR("final|volitile|transient|synchronized|native|abstract|throws"),
		MODIFIER_INHERITENCE("extends|implements"),
		ACTION("return|continue|break|throw|new|assert|strictfp"),
		CONSTRUCT_BLOCK("if|else|do|while|switch|case:|default|for|try|catch|finally"),
		CONSTRUCT_TYPE_DEF("class|interface|enum"),
		SPECIAL_ORGANIZATION("package"),
		SPECIAL_RESERVED("goto|const"),
		SPECIAL_IMPORT("import( +static)?( )+([a-zA-Z\\._])+\\*?"),
		REFERNCE("[a-zA-Z]+[a-zA-Z\\d]*"),
		UNSUPPORTED(LITERAL_LONG, LITERAL_FLOAT, LITERAL_DOUBLE, LITERAL_CHAR,
				LITERAL_STRING, "\\/", MODIFIER_INHERITENCE, SPECIAL_RESERVED,
				"try|catch|finally|enum|interface|assert|new|throw", SPECIAL_IMPORT,
				CONSTRUCT_GENERICS_DEF,
				"instanceof|double|float|volitile|transient|synchronized|native|abstract|throws|:");
		
		public String regex;
		
		private Type(String regex)
		{
			this.regex = regex;
		}
		
		private Type(Object first, Object... objects)
		{
			this.regex = compoundRegex(first, objects);
		}
		
		private Type(String[] strings)
		{
			this(strings[0], (Object[]) Arrays.copyOfRange(strings, 1, strings.length));
		}
		
		public boolean matches(String token)
		{
{'fraction_non_alphanumeric': 0.22400500104188373, 'fraction_numerical': 0.001250260470931444, 'mean_word_length': 5.22568093385214, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7218468', 'n_tokens_mistral': 2034, 'n_tokens_neox': 1847, 'n_words': 215}