List < List < IWord > > sentenceTokenize ( List < Sentence > sentence ) throws IOException { List < List < IWord >> senWords = new ArrayList < List < IWord > > ( ) ; for ( Sentence sen : sentence ) { List < IWord > words = new ArrayList < IWord > ( ) ; wordSeg . reset ( new StringReader ( sen . getValue ( ) ) ) ; IWord word = null ; while ( ( word = wordSeg . next ( ) ) != null ) { words . add ( word ) ; } senWords . add ( words ) ; } return senWords ; }
sentence to words