// Path: src/main/java/com/github/lunatrius/ingameinfo/parser/text/Token.java
// public static enum TokenType {
//     EOF("$"),
//     FUNC_HEAD("<"),
//     FUNC_TAIL(">"),
//     ARGS_HEAD("\\["),
//     ARGS_SEPARATOR("/"),
//     ARGS_TAIL("\\]"),
//     NEWLINE("\\n+"),
//     STRING("(\\\\[<>\\[/\\]\\\\]|[^<>\\[/\\]\\n])+");
// 
//     public final static EnumSet<TokenType> EXCEPTIONS = EnumSet.of(FUNC_TAIL, ARGS_HEAD, ARGS_SEPARATOR, ARGS_TAIL);
//     private final Pattern pattern;
// 
//     private TokenType(final String regex) {
//         this.pattern = Pattern.compile("^" + regex);
//     }
// 
//     public Pattern getPattern() {
//         return this.pattern;
//     }
// }


import java.util.LinkedList;
import java.util.Locale;
import java.util.Queue;
import java.util.regex.Matcher;
import static com.github.lunatrius.ingameinfo.parser.text.Token.TokenType;
package com.github.lunatrius.ingameinfo.parser.text;



public class Tokenizer {
    private final Queue<Token> tokens;

    public Tokenizer() {
        this.tokens = new LinkedList<Token>();
    }

    public void tokenize(String str) throws Exception {
        Location start = new Location(0, 0);
        final Location end = new Location(0, 0);
        boolean match;

        this.tokens.clear();

        str = str.trim();

        while (str.length() > 0) {
            match = false;
// Path: src/main/java/com/github/lunatrius/ingameinfo/parser/text/Token.java
// public static enum TokenType {
//     EOF("$"),
//     FUNC_HEAD("<"),
//     FUNC_TAIL(">"),
//     ARGS_HEAD("\\["),
//     ARGS_SEPARATOR("/"),
//     ARGS_TAIL("\\]"),
//     NEWLINE("\\n+"),
//     STRING("(\\\\[<>\\[/\\]\\\\]|[^<>\\[/\\]\\n])+");
// 
//     public final static EnumSet<TokenType> EXCEPTIONS = EnumSet.of(FUNC_TAIL, ARGS_HEAD, ARGS_SEPARATOR, ARGS_TAIL);
//     private final Pattern pattern;
// 
//     private TokenType(final String regex) {
//         this.pattern = Pattern.compile("^" + regex);
//     }
// 
//     public Pattern getPattern() {
//         return this.pattern;
//     }
// }


// Path: src/main/java/com/github/lunatrius/ingameinfo/parser/text/Tokenizer.java
import java.util.LinkedList;
import java.util.Locale;
import java.util.Queue;
import java.util.regex.Matcher;
import static com.github.lunatrius.ingameinfo.parser.text.Token.TokenType;

package com.github.lunatrius.ingameinfo.parser.text;



public class Tokenizer {
    private final Queue<Token> tokens;

    public Tokenizer() {
        this.tokens = new LinkedList<Token>();
    }

    public void tokenize(String str) throws Exception {
        Location start = new Location(0, 0);
        final Location end = new Location(0, 0);
        boolean match;

        this.tokens.clear();

        str = str.trim();

        while (str.length() > 0) {
            match = false;
            for (final TokenType tokenType : TokenType.values()) {