protected static List < Token > baseTokenize ( String text ) { String [ ] words = text . split ( " " ) ; List < Token > tokens = new LinkedList < Token > ( ) ; for ( String word : words ) { tokens . add ( new Token ( word ) ) ; } return tokens ; }
Split the text on spaces and convert each word into a Token