private void tokenize ( JCas jcas ) { // read tokenized text to add tokens to the jcas Logger . printDetail ( component , "TreeTagger (tokenization) with: " + ttprops . abbFileName ) ; EnumSet < Flag > flags = Flag . getSet ( ttprops . languageSwitch ) ; TreeTaggerTokenizer ttt ; ttprops . abbFileName = "english-abbreviations" ; if ( ttprops . abbFileName != null ) { ttt = new TreeTaggerTokenizer ( ttprops . rootPath + ttprops . fileSeparator + "lib" + ttprops . fileSeparator + ttprops . abbFileName , flags ) ; } else { ttt = new TreeTaggerTokenizer ( null , flags ) ; } String docText = jcas . getDocumentText ( ) . replaceAll ( "\n\n" , "\nEMPTYLINE\n" ) ; List < String > tokenized = ttt . tokenize ( docText ) ; int tokenOffset = 0 ; // loop through all the lines in the treetagger output for ( String s : tokenized ) { // charset missmatch fallback: signal (invalid) s if ( ( ! ( s . equals ( "EMPTYLINE" ) ) ) && ( jcas . getDocumentText ( ) . indexOf ( s , tokenOffset ) < 0 ) ) { Logger . printError ( component , "Tokenization was interrupted because the token \"" + s + "\" could not be found in the original text. The reason for this might be " + "that the encoding of the document is not UTF-8. This token was skipped and " + "if it was part of a temporal expression, will not be extracted." ) ; continue ; } // create tokens and add them to the jcas's indexes. Token newToken = new Token ( jcas ) ; if ( s . equals ( "EMPTYLINE" ) ) { newToken . setBegin ( tokenOffset ) ; newToken . setEnd ( tokenOffset ) ; newToken . setPos ( "EMPTYLINE" ) ; if ( annotate_partofspeech ) { newToken . addToIndexes ( ) ; } } else { newToken . setBegin ( jcas . getDocumentText ( ) . indexOf ( s , tokenOffset ) ) ; newToken . setEnd ( newToken . getBegin ( ) + s . length ( ) ) ; newToken . addToIndexes ( ) ; tokenOffset = newToken . getEnd ( ) ; } } }
tokenizes a given JCas object s document text using the treetagger program and adds the recognized tokens to the JCas object .