public static LangProfile load ( String lang , File file ) { LangProfile profile = new LangProfile ( lang ) ; try ( InputStream is = file . getName ( ) . endsWith ( ".gz" ) ? new GZIPInputStream ( new BufferedInputStream ( new FileInputStream ( file ) ) ) : new BufferedInputStream ( new FileInputStream ( file ) ) ) { TagExtractor tagextractor = new TagExtractor ( "abstract" , 100 ) ; XMLStreamReader reader = null ; try { XMLInputFactory factory = XMLInputFactory . newInstance ( ) ; reader = factory . createXMLStreamReader ( is ) ; while ( reader . hasNext ( ) ) { switch ( reader . next ( ) ) { case XMLStreamReader . START_ELEMENT : tagextractor . setTag ( reader . getName ( ) . toString ( ) ) ; break ; case XMLStreamReader . CHARACTERS : tagextractor . add ( reader . getText ( ) ) ; break ; case XMLStreamReader . END_ELEMENT : tagextractor . closeTag ( profile ) ; break ; } } } catch ( XMLStreamException e ) { throw new RuntimeException ( "Training database file '" + file . getName ( ) + "' is an invalid XML." , e ) ; } finally { try { if ( reader != null ) reader . close ( ) ; } catch ( XMLStreamException e ) { /* ignore exception */ } } logger . info ( lang + ":" + tagextractor . count ( ) ) ; } catch ( IOException e ) { throw new RuntimeException ( "Can't open training database file '" + file . getName ( ) + "'" , e ) ; } return profile ; }
Load Wikipedia abstract database file and generate its language profile