<commit_before>package de.unistuttgart.ims.uimautil;

import org.apache.uima.UimaContext;
import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
import org.apache.uima.fit.component.JCasAnnotator_ImplBase;
import org.apache.uima.fit.descriptor.ConfigurationParameter;
import org.apache.uima.fit.factory.AnnotationFactory;
import org.apache.uima.fit.util.JCasUtil;
import org.apache.uima.jcas.JCas;
import org.apache.uima.jcas.tcas.Annotation;
import org.apache.uima.resource.ResourceInitializationException;

public class ConvertBoundaryToSegmentAnnotation extends JCasAnnotator_ImplBase {

	public static final String PARAM_SEGMENT_ANNOTATION_TYPE = "Segment Annotation Type";
	public static final String PARAM_BOUNDARY_ANNOTATION_TYPE = "Boundary Annotation Type";

	@ConfigurationParameter(name = PARAM_SEGMENT_ANNOTATION_TYPE)
	String segmentAnnotationTypeName;

	@ConfigurationParameter(name = PARAM_BOUNDARY_ANNOTATION_TYPE)
	String boundaryAnnotationTypeName;

	Class<? extends Annotation> segmentAnnotationType;

	Class<? extends Annotation> boundaryAnnotationType;

	@SuppressWarnings("unchecked")
	@Override
	public void initialize(final UimaContext context) throws ResourceInitializationException {
		super.initialize(context);
		Class<?> clazz;
		try {
			clazz = Class.forName(segmentAnnotationTypeName);
			segmentAnnotationType = (Class<? extends Annotation>) clazz;
		} catch (final ClassNotFoundException e) {
			throw new ResourceInitializationException(e);
		}

		try {
			clazz = Class.forName(boundaryAnnotationTypeName);
			boundaryAnnotationType = (Class<? extends Annotation>) clazz;
		} catch (final ClassNotFoundException e) {
			throw new ResourceInitializationException(e);
		}

	}

	@Override
	public void process(JCas jcas) throws AnalysisEngineProcessException {
		int i = 0;
		int index = 0;
		for (final Annotation anno : JCasUtil.select(jcas, boundaryAnnotationType)) {
			if (index > 0)
				AnnotationFactory.createAnnotation(jcas, i, anno.getBegin(), segmentAnnotationType);
			i = anno.getBegin();
			index++;
		}
	}

}<commit_msg>now we can specify how to handle parts before the first and after the last boundary<commit_after>package de.unistuttgart.ims.uimautil;

import org.apache.uima.UimaContext;
import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
import org.apache.uima.fit.component.JCasAnnotator_ImplBase;
import org.apache.uima.fit.descriptor.ConfigurationParameter;
import org.apache.uima.fit.factory.AnnotationFactory;
import org.apache.uima.fit.util.JCasUtil;
import org.apache.uima.jcas.JCas;
import org.apache.uima.jcas.tcas.Annotation;
import org.apache.uima.resource.ResourceInitializationException;

public class ConvertBoundaryToSegmentAnnotation extends JCasAnnotator_ImplBase {

	public static final String PARAM_SEGMENT_ANNOTATION_TYPE = "Segment Annotation Type";
	public static final String PARAM_BOUNDARY_ANNOTATION_TYPE = "Boundary Annotation Type";
	public static final String PARAM_BEGIN_END = "Add segments for the entire document";

	@ConfigurationParameter(name = PARAM_SEGMENT_ANNOTATION_TYPE)
	String segmentAnnotationTypeName;

	@ConfigurationParameter(name = PARAM_BOUNDARY_ANNOTATION_TYPE)
	String boundaryAnnotationTypeName;

	@ConfigurationParameter(name = PARAM_BEGIN_END, mandatory = false)
	boolean beginAndSegment = false;

	Class<? extends Annotation> segmentAnnotationType;

	Class<? extends Annotation> boundaryAnnotationType;

	@SuppressWarnings("unchecked")
	@Override
	public void initialize(final UimaContext context) throws ResourceInitializationException {
		super.initialize(context);
		Class<?> clazz;
		try {
			clazz = Class.forName(segmentAnnotationTypeName);
			segmentAnnotationType = (Class<? extends Annotation>) clazz;
		} catch (final ClassNotFoundException e) {
			throw new ResourceInitializationException(e);
		}

		try {
			clazz = Class.forName(boundaryAnnotationTypeName);
			boundaryAnnotationType = (Class<? extends Annotation>) clazz;
		} catch (final ClassNotFoundException e) {
			throw new ResourceInitializationException(e);
		}

	}

	@Override
	public void process(JCas jcas) throws AnalysisEngineProcessException {
		int i = 0;
		int index = 0;
		for (final Annotation anno : JCasUtil.select(jcas, boundaryAnnotationType)) {
			if ((index > 0) || beginAndSegment)
				AnnotationFactory.createAnnotation(jcas, i, anno.getBegin(), segmentAnnotationType);
			i = anno.getBegin();
			index++;
		}
		if (beginAndSegment)
			AnnotationFactory.createAnnotation(jcas, i, jcas.getDocumentText().length(), segmentAnnotationType);
	}

}
{'fraction_non_alphanumeric': 0.08677595628415301, 'fraction_numerical': 0.0013114754098360656, 'mean_word_length': 6.217665615141956, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7491250', 'n_tokens_mistral': 1438, 'n_tokens_neox': 1330, 'n_words': 296}