protected void validateFeatureDictionary ( ) { FeatureDictionary dict = getFeatureDictionary ( ) ; if ( dict != null ) { if ( dict instanceof Iterable < ? > ) { FeatureDictionary posDict = ( FeatureDictionary ) dict ; Set < String > dictTags = new HashSet < String > ( ) ; Set < String > poisoned = new HashSet < String > ( ) ; for ( WordTag wt : ( Iterable < WordTag > ) posDict ) { dictTags . add ( wt . getPostag ( ) ) ; } Set < String > modelTags = new HashSet < String > ( ) ; AbstractModel posModel = this . artifactProvider . getArtifact ( FeaturizerModel . FEATURIZER_MODEL_ENTRY_NAME ) ; for ( int i = 0 ; i < posModel . getNumOutcomes ( ) ; i ++ ) { modelTags . add ( posModel . getOutcome ( i ) ) ; } for ( String d : dictTags ) { if ( ! modelTags . contains ( d ) ) { poisoned . add ( d ) ; } } this . poisonedDictionaryTags = Collections . unmodifiableSet ( poisoned ) ; // if (poisonedDictionaryTags.size() > 0) { // System.err // .println("WARNING: Feature dictioinary contains tags which are unkown by the model! " // + this.poisonedDictionaryTags.toString()); // } } } }
because the poisoned tags are persisted ...