public Span [ ] tokenizePos ( String d ) { Span [ ] tokens = split ( d ) ; newTokens . clear ( ) ; tokProbs . clear ( ) ; for ( int i = 0 , il = tokens . length ; i < il ; i ++ ) { Span s = tokens [ i ] ; String tok = d . substring ( s . getStart ( ) , s . getEnd ( ) ) ; // Can't tokenize single characters if ( tok . length ( ) < 2 ) { newTokens . add ( s ) ; tokProbs . add ( ONE ) ; } else if ( useAlphaNumericOptimization ( ) && alphaNumeric . matcher ( tok ) . matches ( ) ) { newTokens . add ( s ) ; tokProbs . add ( ONE ) ; } else { int start = s . getStart ( ) ; int end = s . getEnd ( ) ; final int origStart = s . getStart ( ) ; double tokenProb = 1.0 ; for ( int j = origStart + 1 ; j < end ; j ++ ) { double [ ] probs = model . eval ( cg . getContext ( new ObjectIntPair ( tok , j - origStart ) ) ) ; String best = model . getBestOutcome ( probs ) ; //System.err.println("TokenizerME: "+tok.substring(0,j-origStart)+"^"+tok.substring(j-origStart)+" "+best+" "+probs[model.getIndex(best)]); tokenProb *= probs [ model . getIndex ( best ) ] ; if ( best . equals ( TokContextGenerator . SPLIT ) ) { newTokens . add ( new Span ( start , j ) ) ; tokProbs . add ( new Double ( tokenProb ) ) ; start = j ; tokenProb = 1.0 ; } } newTokens . add ( new Span ( start , end ) ) ; tokProbs . add ( new Double ( tokenProb ) ) ; } } Span [ ] spans = new Span [ newTokens . size ( ) ] ; newTokens . toArray ( spans ) ; return spans ; }
Tokenizes the string .