// Path: nicta-ner-common/src/main/java/org/t3as/ner/Token.java
// @Immutable
// public class Token {
// 
//     /** 0-indexed position of the first character of this token from the original text being analysed. */
//     public final int startIndex;
//     /** The text that this token is representing. */
//     public final String text;
// 
//     @JsonCreator
//     public Token(@JsonProperty("startIndex") final int startIndex, @JsonProperty("text") final String text) {
//         this.startIndex = startIndex;
//         this.text = text;
//     }
// 
//     @Override
//     public boolean equals(final Object o) {
//         if (this == o) return true;
//         if (o == null || getClass() != o.getClass()) return false;
//         final Token token = (Token) o;
//         return startIndex == token.startIndex && text.equals(token.text);
//     }
// 
//     @Override
//     public int hashCode() {
//         int result = startIndex;
//         result = 31 * result + text.hashCode();
//         return result;
//     }
// 
//     public String toString() { return startIndex + ":" + text; }
// }


import org.t3as.ner.Token;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import java.util.List;
import static com.google.common.collect.ImmutableList.of;
                       t(20, ":"), t(22, "foo")))},


                {"Dr Foo, Dr. Foo. Mr Bar, Miss Bar, Ms Bar.",
                 of(of(t(0, "Dr"), t(3, "Foo"), t(6, ","), t(8, "Dr."), t(12, "Foo"), t(15, ".")),
                    of(t(17, "Mr"), t(20, "Bar"), t(23, ","), t(25, "Miss"), t(30, "Bar"), t(33, ","), t(35, "Ms"),
                       t(38, "Bar"), t(41, ".")))},

                {"A day in Jan. Jan 2, 2008.",
                 of(of(t(0, "A"), t(2, "day"), t(6, "in"), t(9, "Jan."), t(14, "Jan"), t(18, "2"), t(19, ","),
                       t(21, "2008"), t(25, ".")))},

                {"", of()},

                {":", of(of(t(0, ":")))},

                {"foo:", of(of(t(0, "foo"), t(3, ":")))},

                {"a: b", of(of(t(0, "a"), t(1, ":"), t(3, "b")))},
        };
    }

    @Test
    public void testDataProviders() {
        text();
    }

    @Test(dataProvider = "text")
    public void testProcess(final String text, final List<List<Token>> expected) {
        final Tokenizer t = new Tokenizer(Tokenizer.Mode.WITH_PUNCTUATION);
// Path: nicta-ner-common/src/main/java/org/t3as/ner/Token.java
// @Immutable
// public class Token {
// 
//     /** 0-indexed position of the first character of this token from the original text being analysed. */
//     public final int startIndex;
//     /** The text that this token is representing. */
//     public final String text;
// 
//     @JsonCreator
//     public Token(@JsonProperty("startIndex") final int startIndex, @JsonProperty("text") final String text) {
//         this.startIndex = startIndex;
//         this.text = text;
//     }
// 
//     @Override
//     public boolean equals(final Object o) {
//         if (this == o) return true;
//         if (o == null || getClass() != o.getClass()) return false;
//         final Token token = (Token) o;
//         return startIndex == token.startIndex && text.equals(token.text);
//     }
// 
//     @Override
//     public int hashCode() {
//         int result = startIndex;
//         result = 31 * result + text.hashCode();
//         return result;
//     }
// 
//     public String toString() { return startIndex + ":" + text; }
// }


// Path: nicta-ner/src/test/java/org/t3as/ner/util/TokenizerTest.java
import org.t3as.ner.Token;
import org.testng.Assert;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import java.util.List;
import static com.google.common.collect.ImmutableList.of;

                       t(20, ":"), t(22, "foo")))},


                {"Dr Foo, Dr. Foo. Mr Bar, Miss Bar, Ms Bar.",
                 of(of(t(0, "Dr"), t(3, "Foo"), t(6, ","), t(8, "Dr."), t(12, "Foo"), t(15, ".")),
                    of(t(17, "Mr"), t(20, "Bar"), t(23, ","), t(25, "Miss"), t(30, "Bar"), t(33, ","), t(35, "Ms"),
                       t(38, "Bar"), t(41, ".")))},

                {"A day in Jan. Jan 2, 2008.",
                 of(of(t(0, "A"), t(2, "day"), t(6, "in"), t(9, "Jan."), t(14, "Jan"), t(18, "2"), t(19, ","),
                       t(21, "2008"), t(25, ".")))},

                {"", of()},

                {":", of(of(t(0, ":")))},

                {"foo:", of(of(t(0, "foo"), t(3, ":")))},

                {"a: b", of(of(t(0, "a"), t(1, ":"), t(3, "b")))},
        };
    }

    @Test
    public void testDataProviders() {
        text();
    }

    @Test(dataProvider = "text")
    public void testProcess(final String text, final List<List<Token>> expected) {
        final Tokenizer t = new Tokenizer(Tokenizer.Mode.WITH_PUNCTUATION);
        final List<List<Token>> result = t.process(text);