<commit_before>package pipelines;

import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngine;
import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription;
import static org.apache.uima.fit.factory.ExternalResourceFactory.createExternalResourceDescription;
import static org.apache.uima.fit.pipeline.SimplePipeline.runPipeline;

import java.io.File;

import org.apache.uima.analysis_engine.AnalysisEngine;
import org.apache.uima.analysis_engine.AnalysisEngineDescription;
import org.apache.uima.jcas.JCas;
import org.apache.uima.resource.ExternalResourceDescription;

import resources.AnnotatedCollection_Impl;
import resources.FeaturesList_Impl;
import annotators.FeaturesAnnotator;
import annotators.WekaModelBuilder;

public class ModelCreator {
	
	public static void main(String[] args) throws Exception {
		ExternalResourceDescription candidatesResourceDesc = createExternalResourceDescription(AnnotatedCollection_Impl.class,
				 "file:target/candidates.txt");
		ExternalResourceDescription featuresResourceDesc = createExternalResourceDescription(FeaturesList_Impl.class,
		         new File("features.bin"));
		System.out.println("Loaded");
		
		AnalysisEngineDescription ae_Features = createEngineDescription(FeaturesAnnotator.class,
		        FeaturesAnnotator.CANDIDATE_KEY,
		        candidatesResourceDesc,
		        FeaturesAnnotator.FEATURES_KEY,
		        featuresResourceDesc);
		
		AnalysisEngineDescription ae_Model = createEngineDescription(WekaModelBuilder.class,
		        WekaModelBuilder.FEATURES_KEY,
		        featuresResourceDesc);
		
		AnalysisEngineDescription aed = createEngineDescription(ae_Features, ae_Model);
		
		AnalysisEngine ae = createEngine(aed);
		
		JCas jcas = ae.newJCas();
		
		runPipeline(jcas, ae);
	  }

}<commit_msg>Update the pipeline<commit_after>package pipelines;

import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngine;
import static org.apache.uima.fit.factory.AnalysisEngineFactory.createEngineDescription;
import static org.apache.uima.fit.factory.CollectionReaderFactory.createReader;
import static org.apache.uima.fit.factory.ExternalResourceFactory.createExternalResourceDescription;
import static org.apache.uima.fit.pipeline.SimplePipeline.runPipeline;

import java.io.File;

import org.apache.uima.analysis_engine.AnalysisEngine;
import org.apache.uima.analysis_engine.AnalysisEngineDescription;
import org.apache.uima.collection.CollectionReader;
import org.apache.uima.jcas.JCas;
import org.apache.uima.resource.ExternalResourceDescription;

import readers.CandidateReader;
import resources.AnnotatedCollection_Impl;
import resources.FeaturesList_Impl;
import annotators.FeaturesAnnotator;
import annotators.WekaModelBuilder;

public class ModelCreator {
	
	public static void main(String[] args) throws Exception {
		ExternalResourceDescription candidatesResourceDesc = createExternalResourceDescription(AnnotatedCollection_Impl.class,
				 "file:target/candidates.txt");
		ExternalResourceDescription featuresResourceDesc = createExternalResourceDescription(FeaturesList_Impl.class,
		         new File("features.bin"));
		System.out.println("Loaded");
		
		CollectionReader ae_Reader = createReader(CandidateReader.class,
				CandidateReader.CANDIDATE_KEY,
		        candidatesResourceDesc,
		        CandidateReader.PARAM_DIRECTORY,
		        "train/keys/"
				);
		
		AnalysisEngineDescription ae_Features = createEngineDescription(FeaturesAnnotator.class);
		
		AnalysisEngineDescription ae_Model = createEngineDescription(WekaModelBuilder.class);
		
		AnalysisEngineDescription aed = createEngineDescription(ae_Features, ae_Model);
		
		AnalysisEngine ae = createEngine(aed);
		
		runPipeline(ae_Reader, ae);
	  }

}
{'fraction_non_alphanumeric': 0.07692307692307693, 'fraction_numerical': 0.0, 'mean_word_length': 5.998137802607077, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5551662', 'n_tokens_mistral': 1088, 'n_tokens_neox': 981, 'n_words': 159}