String readWord ( BufferedReader raf ) throws IOException { while ( true ) { // check the buffer first if ( wbp < wordsBuffer . length ) { return wordsBuffer [ wbp ++ ] ; } String line = raf . readLine ( ) ; if ( line == null ) { // end of corpus eoc = true ; return null ; } line = line . trim ( ) ; if ( line . length ( ) == 0 ) { continue ; } cache . writeInt ( - 3 ) ; // mark end of sentence wordsBuffer = line . split ( "\\s+" ) ; wbp = 0 ; eoc = false ; } }
Reads a single word from a file assuming space + tab + EOL to be word boundaries