private void processFile ( InputStream is ) throws MPXJException { try { InputStreamReader reader = new InputStreamReader ( is , CharsetHelper . UTF8 ) ; Tokenizer tk = new ReaderTokenizer ( reader ) { @ Override protected boolean startQuotedIsValid ( StringBuilder buffer ) { return buffer . length ( ) == 1 && buffer . charAt ( 0 ) == ' ' ; } } ; tk . setDelimiter ( DELIMITER ) ; ArrayList < String > columns = new ArrayList < String > ( ) ; String nextTokenPrefix = null ; while ( tk . getType ( ) != Tokenizer . TT_EOF ) { columns . clear ( ) ; TableDefinition table = null ; while ( tk . nextToken ( ) == Tokenizer . TT_WORD ) { String token = tk . getToken ( ) ; if ( columns . size ( ) == 0 ) { if ( token . charAt ( 0 ) == ' ' ) { int index = token . lastIndexOf ( ' ' ) ; if ( index != - 1 ) { String headerToken ; if ( token . endsWith ( "-" ) || token . endsWith ( "=" ) ) { headerToken = token ; token = null ; } else { headerToken = token . substring ( 0 , index ) ; token = token . substring ( index + 1 ) ; } RowHeader header = new RowHeader ( headerToken ) ; table = m_tableDefinitions . get ( header . getType ( ) ) ; columns . add ( header . getID ( ) ) ; } } else { if ( token . charAt ( 0 ) == 0 ) { processFileType ( token ) ; } } } if ( table != null && token != null ) { if ( token . startsWith ( "<\"" ) && ! token . endsWith ( "\">" ) ) { nextTokenPrefix = token ; } else { if ( nextTokenPrefix != null ) { token = nextTokenPrefix + DELIMITER + token ; nextTokenPrefix = null ; } columns . add ( token ) ; } } } if ( table != null && columns . size ( ) > 1 ) { // System.out.println(table.getName() + " " + columns.size()); // ColumnDefinition[] columnDefs = table.getColumns(); // int unknownIndex = 1; // for (int xx = 0; xx < columns.size(); xx++) // { // String x = columns.get(xx); // String columnName = xx < columnDefs.length ? (columnDefs[xx] == null ? "UNKNOWN" + (unknownIndex++) : columnDefs[xx].getName()) : "?"; // System.out.println(columnName + ": " + x + ", "); // } // System.out.println(); TextFileRow row = new TextFileRow ( table , columns , m_epochDateFormat ) ; List < Row > rows = m_tables . get ( table . getName ( ) ) ; if ( rows == null ) { rows = new LinkedList < Row > ( ) ; m_tables . put ( table . getName ( ) , rows ) ; } rows . add ( row ) ; } } } catch ( Exception ex ) { throw new MPXJException ( MPXJException . READ_ERROR , ex ) ; } }
Tokenizes the input file and extracts the required data .