public ViterbiLattice build ( String text ) { int textLength = text . length ( ) ; ViterbiLattice lattice = new ViterbiLattice ( textLength + 2 ) ; lattice . addBos ( ) ; int unknownWordEndIndex = - 1 ; // index of the last character of unknown word for ( int startIndex = 0 ; startIndex < textLength ; startIndex ++ ) { // If no token ends where current token starts, skip this index if ( lattice . tokenEndsWhereCurrentTokenStarts ( startIndex ) ) { String suffix = text . substring ( startIndex ) ; boolean found = processIndex ( lattice , startIndex , suffix ) ; // In the case of normal mode, it doesn't process unknown word greedily. if ( searchMode || unknownWordEndIndex <= startIndex ) { int [ ] categories = characterDefinitions . lookupCategories ( suffix . charAt ( 0 ) ) ; for ( int i = 0 ; i < categories . length ; i ++ ) { int category = categories [ i ] ; unknownWordEndIndex = processUnknownWord ( category , i , lattice , unknownWordEndIndex , startIndex , suffix , found ) ; } } } } if ( useUserDictionary ) { processUserDictionary ( text , lattice ) ; } lattice . addEos ( ) ; return lattice ; }
Build lattice from input text