<commit_before>package edu.ucdenver.ccp.nlp.pipelines.runner.serialization;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.GZIPInputStream;

import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.uima.UimaContext;
import org.apache.uima.analysis_engine.AnalysisEngineDescription;
import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
import org.apache.uima.cas.CASException;
import org.apache.uima.fit.component.JCasAnnotator_ImplBase;
import org.apache.uima.fit.descriptor.ConfigurationParameter;
import org.apache.uima.fit.factory.AnalysisEngineFactory;
import org.apache.uima.jcas.JCas;
import org.apache.uima.resource.ResourceInitializationException;
import org.apache.uima.resource.metadata.TypeSystemDescription;
import org.apache.uima.util.Level;
import org.apache.uima.util.Logger;

import edu.ucdenver.ccp.common.file.CharacterEncoding;
import edu.ucdenver.ccp.common.file.reader.Line;
import edu.ucdenver.ccp.common.file.reader.StreamLineIterator;
import edu.ucdenver.ccp.common.reflection.ConstructorUtil;
import edu.ucdenver.ccp.nlp.core.annotation.TextAnnotation;
import edu.ucdenver.ccp.nlp.pipelines.log.ProcessingErrorLog;
import edu.ucdenver.ccp.nlp.pipelines.runner.serialization.AnnotationSerializer.IncludeAnnotator;
import edu.ucdenver.ccp.nlp.pipelines.runner.serialization.AnnotationSerializer.IncludeCoveredText;
import edu.ucdenver.ccp.nlp.uima.shims.ShimDefaults;
import edu.ucdenver.ccp.nlp.uima.util.UIMA_Util;
import edu.ucdenver.ccp.nlp.uima.util.View_Util;
import edu.ucdenver.ccp.uima.shims.document.DocumentMetadataHandler;

public class AnnotationDeserializerAE extends JCasAnnotator_ImplBase {

	public static final String PARAM_LOAD_DIRECTORY = "loadDirectory";
	@ConfigurationParameter(mandatory = false, description = "")
	private File loadDirectory;

	/**
	 * If the outputDirectory is not specified, then this view will be used to
	 * search for the source-document-path, which will then be used as the
	 * serialization directory for the output of this annotator.
	 */
	public static final String PARAM_SOURCE_VIEW_NAME = "sourceViewName";
	@ConfigurationParameter(mandatory = false, description = "")
	private String sourceViewName;

	/**
	 * Signifies the view where the annotations will be loaded
	 */
	public static final String PARAM_LOAD_VIEW_NAME = "loadViewName";
	@ConfigurationParameter(mandatory = false, description = "")
	private String loadViewName;

	/**
	 * If true, the serialized annotations will include the covered text so it
	 * will be imported, default=false
	 */
	public static final String PARAM_INCLUDE_COVERED_TEXT = "includeCoveredText";
	@ConfigurationParameter(mandatory = false, description = "", defaultValue = "false")
	private boolean includeCoveredText;

	public static final String PARAM_INCLUDE_ANNOTATOR = "includeAnnotator";
	@ConfigurationParameter(mandatory = false, description = "", defaultValue = "true")
	private boolean includeAnnotator;

	public static final String PARAM_INPUT_FILENAME_INFIXES = "inputFilenameInfixes";
	@ConfigurationParameter(mandatory = true, description = "An array of infixes to use to identify annotations files to load.")
	private String[] inputFilenameInfixes;

	public static final String PARAM_DOCUMENT_METADATA_HANDLER_CLASS = "documentMetadataHandlerClassName";
	private static final String ANNOT_FILE_SUFFIX = ".annot";
	@ConfigurationParameter(mandatory = false, description = "name of the DocumentMetadataHandler implementation to use", defaultValue = ShimDefaults.CCP_DOCUMENT_METADATA_HANDLER_CLASS_NAME)
	private String documentMetadataHandlerClassName;
	private DocumentMetadataHandler documentMetaDataHandler;

	private Logger logger;

	@Override
	public void initialize(UimaContext aContext) throws ResourceInitializationException {
		super.initialize(aContext);
		documentMetaDataHandler = (DocumentMetadataHandler) ConstructorUtil
				.invokeConstructor(documentMetadataHandlerClassName);
		logger = aContext.getLogger();
	}

	@Override
	public void process(JCas jCas) throws AnalysisEngineProcessException {
		String documentId = documentMetaDataHandler.extractDocumentId(jCas);
		try {
			for (String infix : inputFilenameInfixes) {
				File inputFile = getInputFile(jCas, documentId, infix);<commit_msg>Added log to output list of infixes-to-load<commit_after>package edu.ucdenver.ccp.nlp.pipelines.runner.serialization;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.zip.GZIPInputStream;

import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.uima.UimaContext;
import org.apache.uima.analysis_engine.AnalysisEngineDescription;
import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
import org.apache.uima.cas.CASException;
import org.apache.uima.fit.component.JCasAnnotator_ImplBase;
import org.apache.uima.fit.descriptor.ConfigurationParameter;
import org.apache.uima.fit.factory.AnalysisEngineFactory;
import org.apache.uima.jcas.JCas;
import org.apache.uima.resource.ResourceInitializationException;
import org.apache.uima.resource.metadata.TypeSystemDescription;
import org.apache.uima.util.Level;
import org.apache.uima.util.Logger;

import edu.ucdenver.ccp.common.file.CharacterEncoding;
import edu.ucdenver.ccp.common.file.reader.Line;
import edu.ucdenver.ccp.common.file.reader.StreamLineIterator;
import edu.ucdenver.ccp.common.reflection.ConstructorUtil;
import edu.ucdenver.ccp.nlp.core.annotation.TextAnnotation;
import edu.ucdenver.ccp.nlp.pipelines.log.ProcessingErrorLog;
import edu.ucdenver.ccp.nlp.pipelines.runner.serialization.AnnotationSerializer.IncludeAnnotator;
import edu.ucdenver.ccp.nlp.pipelines.runner.serialization.AnnotationSerializer.IncludeCoveredText;
import edu.ucdenver.ccp.nlp.uima.shims.ShimDefaults;
import edu.ucdenver.ccp.nlp.uima.util.UIMA_Util;
import edu.ucdenver.ccp.nlp.uima.util.View_Util;
import edu.ucdenver.ccp.uima.shims.document.DocumentMetadataHandler;

public class AnnotationDeserializerAE extends JCasAnnotator_ImplBase {

	public static final String PARAM_LOAD_DIRECTORY = "loadDirectory";
	@ConfigurationParameter(mandatory = false, description = "")
	private File loadDirectory;

	/**
	 * If the outputDirectory is not specified, then this view will be used to
	 * search for the source-document-path, which will then be used as the
	 * serialization directory for the output of this annotator.
	 */
	public static final String PARAM_SOURCE_VIEW_NAME = "sourceViewName";
	@ConfigurationParameter(mandatory = false, description = "")
	private String sourceViewName;

	/**
	 * Signifies the view where the annotations will be loaded
	 */
	public static final String PARAM_LOAD_VIEW_NAME = "loadViewName";
	@ConfigurationParameter(mandatory = false, description = "")
	private String loadViewName;

	/**
	 * If true, the serialized annotations will include the covered text so it
	 * will be imported, default=false
	 */
	public static final String PARAM_INCLUDE_COVERED_TEXT = "includeCoveredText";
	@ConfigurationParameter(mandatory = false, description = "", defaultValue = "false")
	private boolean includeCoveredText;

	public static final String PARAM_INCLUDE_ANNOTATOR = "includeAnnotator";
	@ConfigurationParameter(mandatory = false, description = "", defaultValue = "true")
	private boolean includeAnnotator;

	public static final String PARAM_INPUT_FILENAME_INFIXES = "inputFilenameInfixes";
	@ConfigurationParameter(mandatory = true, description = "An array of infixes to use to identify annotations files to load.")
	private String[] inputFilenameInfixes;

	public static final String PARAM_DOCUMENT_METADATA_HANDLER_CLASS = "documentMetadataHandlerClassName";
	private static final String ANNOT_FILE_SUFFIX = ".annot";
	@ConfigurationParameter(mandatory = false, description = "name of the DocumentMetadataHandler implementation to use", defaultValue = ShimDefaults.CCP_DOCUMENT_METADATA_HANDLER_CLASS_NAME)
	private String documentMetadataHandlerClassName;
	private DocumentMetadataHandler documentMetaDataHandler;

	private Logger logger;

	@Override
	public void initialize(UimaContext aContext) throws ResourceInitializationException {
		super.initialize(aContext);
		documentMetaDataHandler = (DocumentMetadataHandler) ConstructorUtil
				.invokeConstructor(documentMetadataHandlerClassName);
		logger = aContext.getLogger();

		logger.log(Level.INFO, "AnnotationDeserializer initialized with the following file infixes: "
				+ Arrays.toString(inputFilenameInfixes));
	}

	@Override
	public void process(JCas jCas) throws AnalysisEngineProcessException {
		String documentId = documentMetaDataHandler.extractDocumentId(jCas);
		try {
			for (String infix : inputFilenameInfixes) {
				File inputFile = getInputFile(jCas, documentId, infix);
{'fraction_non_alphanumeric': 0.08396605627512282, 'fraction_numerical': 0.0, 'mean_word_length': 6.464166666666666, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '265786', 'n_tokens_mistral': 2697, 'n_tokens_neox': 2574, 'n_words': 578}