public String tokenizeToText ( ) { final StringBuilder sb = new StringBuilder ( ) ; if ( isNoSeg ) { String [ ] sentences = text . toArray ( new String [ text . size ( ) ] ) ; final List < List < Token > > tokens = tokenizer . tokenize ( sentences ) ; for ( final List < Token > tokSentence : tokens ) { for ( final Token tok : tokSentence ) { String tokenValue = tok . getTokenValue ( ) ; sb . append ( tokenValue . trim ( ) ) . append ( DELIMITER ) ; } sb . append ( LINE_BREAK ) ; } } else { final String [ ] sentences = segmenter . segmentSentence ( ) ; final List < List < Token > > tokens = tokenizer . tokenize ( sentences ) ; for ( final List < Token > tokSentence : tokens ) { for ( final Token token : tokSentence ) { String tokenValue = token . getTokenValue ( ) ; if ( tokenValue . equals ( RuleBasedSegmenter . PARAGRAPH ) ) { sb . append ( DEFAULT_TOKEN_VALUE ) . append ( LINE_BREAK ) ; } else { sb . append ( tokenValue . trim ( ) ) . append ( DELIMITER ) ; } } sb . append ( LINE_BREAK ) ; } } return sb . toString ( ) . trim ( ) ; }
Tokenize and Segment input text . Outputs tokens in running text format one sentence per line .