public SourcedToken [ ] sourcedTokenize ( String input , String source ) { Token [ ] tokens = tokenize ( input ) ; SourcedToken [ ] sourcedTokens = new SourcedToken [ tokens . length ] ; for ( int i = 0 ; i < tokens . length ; i ++ ) { String key = tokens [ i ] . getValue ( ) + "@" + source ; if ( tokMap . get ( key ) == null ) { tokMap . put ( key , new Integer ( ++ nextId ) ) ; } int id = ( ( Integer ) tokMap . get ( key ) ) . intValue ( ) ; sourcedTokens [ i ] = new BasicSourcedToken ( id , tokens [ i ] . getValue ( ) , source ) ; } return sourcedTokens ; }
Return tokenized version of a string .