// Path: src/com/iconmaster/source/element/Element.java
// public class Element implements IDirectable {
// 	public IElementType type;
// 	public Object[] args = new Object[10];
// 	public Element dataType = null;
// 	public ArrayList<String> directives = new ArrayList<>();
// 	public Range range;
// 
// 	public Element(Range range, IElementType type) {
// 		this.range = range;
// 		this.type = type;
// 	}
// 
// 	@Override
// 	public String toString() {
// 		StringBuilder sb = new StringBuilder("{");
// 		sb.append(type).append(" as ").append(dataType).append(": [").append(args[0]).append(" ").append(args[1]).append(" ").append(args[2]).append(" ").append(args[3]).append("]");
// 		return sb.append("}").toString();
// 	}
// 
// 	@Override
// 	public ArrayList<String> getDirectives() {
// 		return directives;
// 	}
// }
// 
// Path: src/com/iconmaster/source/exception/SourceException.java
// public class SourceException extends Exception {
// 	private final Range range;
// 	
// 	public SourceException(Range range) {
// 		super(range.toString()+": Unknown Source exception");
// 		this.range = range;
// 	}
// 	
// 	public SourceException(Range range, String message) {
// 		super((range==null?"null: ":range.toString()+": ")+message);
// 		this.range = range;
// 	}
// 	
// 	@Override
// 	public String getMessage() {
// 		return super.getMessage();
// 	}
// 	
// 	public Range getRange() {
// 		return range;
// 	}
// }
// 
// Path: src/com/iconmaster/source/util/Range.java
// public class Range {
// 	public final int low;
// 	public final int high;
// 
// 	public Range(int low, int high) {
// 		this.low = Math.min(low, high);
// 		this.high = Math.max(low, high);
// 	}
// 
// 	@Override
// 	public String toString() {
// 		return low+"~"+high;
// 	}
// 	
// 	public static Range from(Range r1, Range r2) {
// 		return new Range(r1==null?0:r1.low,r2==null?0:r2.high);
// 	}
// 	
// 	
// }


import com.iconmaster.source.element.Element;
import com.iconmaster.source.exception.SourceException;
import com.iconmaster.source.util.Range;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
package com.iconmaster.source.tokenize;


/**
 *
 * @author iconmaster
 */
public class Tokenizer {
	public static ArrayList<Element> tokenize(String input) throws SourceException {
		return tokenize(input, true);
	}
	
	public static ArrayList<Element> tokenize(String input, boolean expand) throws SourceException {
		ArrayList<Token> a = new ArrayList<>();
		
		//add basic tokens
		int len = 0;
		while (!input.isEmpty()) {
			//System.out.println(input);
			boolean found = false;
			for (TokenRule rule : TokenRule.values()) {
				Matcher m = Pattern.compile(rule.match).matcher(input);
				if (m.find()) {
					String got = m.group();
					int olen = len;
					len+=got.length();
					if (expand) {
						got = rule.format(got);
					}
					input = m.replaceFirst("");
					if (got!=null) {
// Path: src/com/iconmaster/source/element/Element.java
// public class Element implements IDirectable {
// 	public IElementType type;
// 	public Object[] args = new Object[10];
// 	public Element dataType = null;
// 	public ArrayList<String> directives = new ArrayList<>();
// 	public Range range;
// 
// 	public Element(Range range, IElementType type) {
// 		this.range = range;
// 		this.type = type;
// 	}
// 
// 	@Override
// 	public String toString() {
// 		StringBuilder sb = new StringBuilder("{");
// 		sb.append(type).append(" as ").append(dataType).append(": [").append(args[0]).append(" ").append(args[1]).append(" ").append(args[2]).append(" ").append(args[3]).append("]");
// 		return sb.append("}").toString();
// 	}
// 
// 	@Override
// 	public ArrayList<String> getDirectives() {
// 		return directives;
// 	}
// }
// 
// Path: src/com/iconmaster/source/exception/SourceException.java
// public class SourceException extends Exception {
// 	private final Range range;
// 	
// 	public SourceException(Range range) {
// 		super(range.toString()+": Unknown Source exception");
// 		this.range = range;
// 	}
// 	
// 	public SourceException(Range range, String message) {
// 		super((range==null?"null: ":range.toString()+": ")+message);
// 		this.range = range;
// 	}
// 	
// 	@Override
// 	public String getMessage() {
// 		return super.getMessage();
// 	}
// 	
// 	public Range getRange() {
// 		return range;
// 	}
// }
// 
// Path: src/com/iconmaster/source/util/Range.java
// public class Range {
// 	public final int low;
// 	public final int high;
// 
// 	public Range(int low, int high) {
// 		this.low = Math.min(low, high);
// 		this.high = Math.max(low, high);
// 	}
// 
// 	@Override
// 	public String toString() {
// 		return low+"~"+high;
// 	}
// 	
// 	public static Range from(Range r1, Range r2) {
// 		return new Range(r1==null?0:r1.low,r2==null?0:r2.high);
// 	}
// 	
// 	
// }


// Path: src/com/iconmaster/source/tokenize/Tokenizer.java
import com.iconmaster.source.element.Element;
import com.iconmaster.source.exception.SourceException;
import com.iconmaster.source.util.Range;
import java.util.ArrayList;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

package com.iconmaster.source.tokenize;


/**
 *
 * @author iconmaster
 */
public class Tokenizer {
	public static ArrayList<Element> tokenize(String input) throws SourceException {
		return tokenize(input, true);
	}
	
	public static ArrayList<Element> tokenize(String input, boolean expand) throws SourceException {
		ArrayList<Token> a = new ArrayList<>();
		
		//add basic tokens
		int len = 0;
		while (!input.isEmpty()) {
			//System.out.println(input);
			boolean found = false;
			for (TokenRule rule : TokenRule.values()) {
				Matcher m = Pattern.compile(rule.match).matcher(input);
				if (m.find()) {
					String got = m.group();
					int olen = len;
					len+=got.length();
					if (expand) {
						got = rule.format(got);
					}
					input = m.replaceFirst("");
					if (got!=null) {
						a.add(new Token(new Range(olen,len),rule,got));