<commit_before>package edu.cmu.ml.rtw.micro.model.annotation.nlp;

import java.util.ArrayList;
import java.util.List;

import edu.cmu.ml.rtw.generic.model.annotator.nlp.PipelineNLP;
import edu.cmu.ml.rtw.micro.hdp.HDPParser;
import edu.cmu.ml.rtw.micro.cat.data.CatDataTools;
import edu.cmu.ml.rtw.micro.cat.data.annotation.CategoryList;
import edu.cmu.ml.rtw.micro.cat.data.annotation.nlp.NELLMentionCategorizer;
import edu.cmu.ml.rtw.micro.sem.model.annotation.nlp.SemparseAnnotatorSentence;
import edu.cmu.ml.rtw.ppa.predict.PPADisambiguator;
import edu.cmu.ml.rtw.AnnotationVerb;

public class PipelineNLPMicro extends PipelineNLP {
	public enum Annotator {
		NP_CATEGORIZER,
		VERB_ANNOTATOR,
		SEMANTIC_PARSER,
		HDP_PARSER,
		PPA_DISAMBIGUATOR
	}
	
	public PipelineNLPMicro() {
		this(NELLMentionCategorizer.DEFAULT_MENTION_MODEL_THRESHOLD, null);
	}
	
	public PipelineNLPMicro(List<Annotator> disabledAnnotators) {
		this(NELLMentionCategorizer.DEFAULT_MENTION_MODEL_THRESHOLD, disabledAnnotators);
	}
	
	public PipelineNLPMicro(double nounPhraseMentionModelThreshold) {
		this(nounPhraseMentionModelThreshold, null);
	}
	
	public PipelineNLPMicro(double nounPhraseMentionModelThreshold, List<Annotator> disabledAnnotators) {
		super();

		if (disabledAnnotators == null)
			disabledAnnotators = new ArrayList<Annotator>();
		
		/*
		 * Initialize and add micro-readers here
		 */
		if (disabledAnnotators.contains(Annotator.NP_CATEGORIZER))
			throw new IllegalArgumentException("Cannot disable noun-phrase categorizer.");
		
		NELLMentionCategorizer mentionCategorizer = new NELLMentionCategorizer(
				new CategoryList(CategoryList.Type.ALL_NELL_CATEGORIES, new CatDataTools()), 
				nounPhraseMentionModelThreshold, NELLMentionCategorizer.DEFAULT_LABEL_TYPE, 
				1);
		addAnnotator(mentionCategorizer.produces(), mentionCategorizer);
		
		
		if (!disabledAnnotators.contains(Annotator.SEMANTIC_PARSER)) {
			SemparseAnnotatorSentence semanticParser = SemparseAnnotatorSentence.fromSerializedModels(SemparseAnnotatorSentence.PARSER_MODEL_PATH, SemparseAnnotatorSentence.SUPERTAGGER_MODEL_PATH);
			addAnnotator(semanticParser.produces(), semanticParser);		
		}
		
		if (!disabledAnnotators.contains(Annotator.VERB_ANNOTATOR)) {
			AnnotationVerb annotationVerb = new AnnotationVerb();
			addAnnotator(annotationVerb.produces(), annotationVerb);
		}
		
		if (!disabledAnnotators.contains(Annotator.PPA_DISAMBIGUATOR)) {
			PPADisambiguator ppa = new PPADisambiguator();	
			addAnnotator(ppa.produces(), ppa);
		}
		
		if (!disabledAnnotators.contains(Annotator.HDP_PARSER)) {
			HDPParser hdpParser = HDPParser.getInstance();
			addAnnotator(hdpParser.produces(), hdpParser);
		}
	}
	
	public PipelineNLPMicro(PipelineNLPMicro pipeline) {
		this.annotationOrder = pipeline.annotationOrder;
		this.annotators = pipeline.annotators;
		this.document = pipeline.document;
	}
}<commit_msg>* Quick and dirty tweaks for mass-microreading for KBP 2015, esp. multithreading and being able to skip micro-cat<commit_after>package edu.cmu.ml.rtw.micro.model.annotation.nlp;

import java.util.ArrayList;
import java.util.List;

import edu.cmu.ml.rtw.generic.model.annotator.nlp.PipelineNLP;
import edu.cmu.ml.rtw.micro.hdp.HDPParser;
import edu.cmu.ml.rtw.micro.cat.data.CatDataTools;
import edu.cmu.ml.rtw.micro.cat.data.annotation.CategoryList;
import edu.cmu.ml.rtw.micro.cat.data.annotation.nlp.NELLMentionCategorizer;
import edu.cmu.ml.rtw.micro.sem.model.annotation.nlp.SemparseAnnotatorSentence;
import edu.cmu.ml.rtw.ppa.predict.PPADisambiguator;
import edu.cmu.ml.rtw.AnnotationVerb;

public class PipelineNLPMicro extends PipelineNLP {
	public enum Annotator {
		NP_CATEGORIZER,
		VERB_ANNOTATOR,
		SEMANTIC_PARSER,
		HDP_PARSER,
		PPA_DISAMBIGUATOR
	}
	
	public PipelineNLPMicro() {
		this(NELLMentionCategorizer.DEFAULT_MENTION_MODEL_THRESHOLD, null);
	}
	
	public PipelineNLPMicro(List<Annotator> disabledAnnotators) {
		this(NELLMentionCategorizer.DEFAULT_MENTION_MODEL_THRESHOLD, disabledAnnotators);
	}
	
	public PipelineNLPMicro(double nounPhraseMentionModelThreshold) {
		this(nounPhraseMentionModelThreshold, null);
	}
	
	public PipelineNLPMicro(double nounPhraseMentionModelThreshold, List<Annotator> disabledAnnotators) {
		super();

		if (disabledAnnotators == null)
			disabledAnnotators = new ArrayList<Annotator>();
		
		/*
		 * Initialize and add micro-readers here
		 */
		if (disabledAnnotators.contains(Annotator.NP_CATEGORIZER) && false)  //bkdb
			throw new IllegalArgumentException("Cannot disable noun-phrase categorizer.");

                if (!disabledAnnotators.contains(Annotator.NP_CATEGORIZER)) {
                    NELLMentionCategorizer mentionCategorizer = new NELLMentionCategorizer(
                            new CategoryList(CategoryList.Type.ALL_NELL_CATEGORIES, new CatDataTools()), 
                            nounPhraseMentionModelThreshold, NELLMentionCategorizer.DEFAULT_LABEL_TYPE, 
                            20);
                    addAnnotator(mentionCategorizer.produces(), mentionCategorizer);
                }
		
		if (!disabledAnnotators.contains(Annotator.SEMANTIC_PARSER)) {
			SemparseAnnotatorSentence semanticParser = SemparseAnnotatorSentence.fromSerializedModels(SemparseAnnotatorSentence.PARSER_MODEL_PATH, SemparseAnnotatorSentence.SUPERTAGGER_MODEL_PATH);
			addAnnotator(semanticParser.produces(), semanticParser);		
		}
		
		if (!disabledAnnotators.contains(Annotator.VERB_ANNOTATOR)) {
			AnnotationVerb annotationVerb = new AnnotationVerb();
			addAnnotator(annotationVerb.produces(), annotationVerb);
		}
		
		if (!disabledAnnotators.contains(Annotator.PPA_DISAMBIGUATOR)) {
			PPADisambiguator ppa = new PPADisambiguator();	
			addAnnotator(ppa.produces(), ppa);
		}
		
		if (!disabledAnnotators.contains(Annotator.HDP_PARSER)) {
			HDPParser hdpParser = HDPParser.getInstance();
			addAnnotator(hdpParser.produces(), hdpParser);
		}
	}
	
	public PipelineNLPMicro(PipelineNLPMicro pipeline) {
		this.annotationOrder = pipeline.annotationOrder;
		this.annotators = pipeline.annotators;
		this.document = pipeline.document;
	}
}
{'fraction_non_alphanumeric': 0.09464548878336336, 'fraction_numerical': 0.0011462256427050926, 'mean_word_length': 5.132530120481928, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3636305', 'n_tokens_mistral': 2295, 'n_tokens_neox': 2116, 'n_words': 275}