<commit_before>package de.unistuttgart.quadrama.io.gutenbergde;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
import java.util.TreeSet;

import org.apache.commons.io.IOUtils;
import org.apache.uima.collection.CollectionException;
import org.apache.uima.fit.factory.AnnotationFactory;
import org.apache.uima.fit.util.JCasUtil;
import org.apache.uima.jcas.JCas;
import org.apache.uima.jcas.tcas.Annotation;
import org.jsoup.Jsoup;

import de.unistuttgart.ims.uimautil.IMSUtil;
import de.unistuttgart.quadrama.api.Act;
import de.unistuttgart.quadrama.api.Drama;
import de.unistuttgart.quadrama.api.DramatisPersonae;
import de.unistuttgart.quadrama.api.Figure;
import de.unistuttgart.quadrama.api.Footnote;
import de.unistuttgart.quadrama.api.FrontMatter;
import de.unistuttgart.quadrama.api.MainMatter;
import de.unistuttgart.quadrama.api.Scene;
import de.unistuttgart.quadrama.api.Speaker;
import de.unistuttgart.quadrama.api.Speech;
import de.unistuttgart.quadrama.api.StageDirection;
import de.unistuttgart.quadrama.api.Utterance;
import de.unistuttgart.quadrama.io.core.AbstractDramaReader;
import de.unistuttgart.quadrama.io.core.type.HTMLAnnotation;

public class GutenbergDEReader extends AbstractDramaReader {

	@Override
	public void getNext(JCas jcas) throws IOException, CollectionException {

		Drama drama = new Drama(jcas);
		drama.setDocumentId("test");
		drama.addToIndexes();
		jcas.setDocumentLanguage(language);

		File file = files[current++];
		getLogger().debug("Processing file " + file.getAbsolutePath());

		String str = IOUtils.toString(new FileInputStream(file));
		org.jsoup.nodes.Document doc = Jsoup.parseBodyFragment(str);
		Visitor vis = new Visitor(jcas);
		doc.traverse(vis);
		jcas = vis.getJCas();
		Map<String, HTMLAnnotation> annoMap = vis.getAnnotationMap();

		// identify front and main matter
		select2Annotation(jcas, doc, annoMap, "div.gutenb:eq(0)",
				FrontMatter.class, null);
		selectRange2Annotation(jcas, doc, annoMap, "div.gutenb:eq(1)",
				"div.gutenb:last-child", MainMatter.class);
		FrontMatter frontMatter =
				JCasUtil.selectSingle(jcas, FrontMatter.class);
		MainMatter mainMatter = JCasUtil.selectSingle(jcas, MainMatter.class);

		// identify simple annotations
		select2Annotation(jcas, doc, annoMap, "span.speaker", Speaker.class,
				mainMatter);
		select2Annotation(jcas, doc, annoMap, "span.speaker", Figure.class,
				frontMatter);
		select2Annotation(jcas, doc, annoMap, "span.regie",
				StageDirection.class, mainMatter);
		select2Annotation(jcas, doc, annoMap, "span.footnote", Footnote.class,
				mainMatter);
		select2Annotation(jcas, doc, annoMap, "h3 + p", DramatisPersonae.class,
				frontMatter);

		select2Annotation(jcas, doc, annoMap, "p:has(span.speaker)",
				Utterance.class, mainMatter);

		annotateSpeech(jcas, mainMatter);

		// aggregating annotations
		// TODO: convert to range function
		int currentSceneBegin = -1;
		int currentActBegin = -1;
		for (HTMLAnnotation anno : JCasUtil.select(jcas, HTMLAnnotation.class)) {
			if (anno.getTag().equals("h2")) {
				if (currentSceneBegin >= 0) {
					AnnotationFactory.createAnnotation(jcas, currentSceneBegin,
							anno.getBegin() - 1, Scene.class);
				}
				currentSceneBegin = anno.getBegin();
			}
			if (anno.getTag().equals("h1")) {
				if (currentActBegin >= 0) {
					AnnotationFactory.createAnnotation(jcas, currentActBegin,
							anno.getBegin() - 1, Act.class);
				}
				currentActBegin = anno.getBegin();
			}
		}

		this.cleanUp(jcas);

	}

	protected void annotateSpeech(JCas jcas, Annotation mainMatter) {
		for (Utterance utterance : JCasUtil.selectCovered(Utterance.class,
				mainMatter)) {
			TreeSet<Annotation> except =
					new TreeSet<Annotation>(new Comparator<Annotation>() {

						public int compare(Annotation o1, Annotation o2) {
							return Integer.compare(o1.getBegin(), o2.getBegin());
						}
<commit_msg>now we include the last scene and act, also producing heading annotations<commit_after>package de.unistuttgart.quadrama.io.gutenbergde;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Map;
import java.util.TreeSet;

import org.apache.commons.io.IOUtils;
import org.apache.uima.collection.CollectionException;
import org.apache.uima.fit.factory.AnnotationFactory;
import org.apache.uima.fit.util.JCasUtil;
import org.apache.uima.jcas.JCas;
import org.apache.uima.jcas.tcas.Annotation;
import org.jsoup.Jsoup;

import de.unistuttgart.ims.uimautil.IMSUtil;
import de.unistuttgart.quadrama.api.Act;
import de.unistuttgart.quadrama.api.ActHeading;
import de.unistuttgart.quadrama.api.Drama;
import de.unistuttgart.quadrama.api.DramatisPersonae;
import de.unistuttgart.quadrama.api.Figure;
import de.unistuttgart.quadrama.api.Footnote;
import de.unistuttgart.quadrama.api.FrontMatter;
import de.unistuttgart.quadrama.api.MainMatter;
import de.unistuttgart.quadrama.api.Scene;
import de.unistuttgart.quadrama.api.SceneHeading;
import de.unistuttgart.quadrama.api.Speaker;
import de.unistuttgart.quadrama.api.Speech;
import de.unistuttgart.quadrama.api.StageDirection;
import de.unistuttgart.quadrama.api.Utterance;
import de.unistuttgart.quadrama.io.core.AbstractDramaReader;
import de.unistuttgart.quadrama.io.core.type.HTMLAnnotation;

public class GutenbergDEReader extends AbstractDramaReader {

	@Override
	public void getNext(JCas jcas) throws IOException, CollectionException {

		Drama drama = new Drama(jcas);
		drama.setDocumentId("test");
		drama.addToIndexes();
		jcas.setDocumentLanguage(language);

		File file = files[current++];
		getLogger().debug("Processing file " + file.getAbsolutePath());

		String str = IOUtils.toString(new FileInputStream(file));
		org.jsoup.nodes.Document doc = Jsoup.parseBodyFragment(str);
		Visitor vis = new Visitor(jcas);
		doc.traverse(vis);
		jcas = vis.getJCas();
		Map<String, HTMLAnnotation> annoMap = vis.getAnnotationMap();

		// identify front and main matter
		select2Annotation(jcas, doc, annoMap, "div.gutenb:eq(0)",
				FrontMatter.class, null);
		selectRange2Annotation(jcas, doc, annoMap, "div.gutenb:eq(1)",
				"div.gutenb:last-child", MainMatter.class);
		FrontMatter frontMatter =
				JCasUtil.selectSingle(jcas, FrontMatter.class);
		MainMatter mainMatter = JCasUtil.selectSingle(jcas, MainMatter.class);

		// identify simple annotations
		select2Annotation(jcas, doc, annoMap, "span.speaker", Speaker.class,
				mainMatter);
		select2Annotation(jcas, doc, annoMap, "span.speaker", Figure.class,
				frontMatter);
		select2Annotation(jcas, doc, annoMap, "span.regie",
				StageDirection.class, mainMatter);
		select2Annotation(jcas, doc, annoMap, "span.footnote", Footnote.class,
				mainMatter);
		select2Annotation(jcas, doc, annoMap, "h3 + p", DramatisPersonae.class,
				frontMatter);

		select2Annotation(jcas, doc, annoMap, "p:has(span.speaker)",
				Utterance.class, mainMatter);

		annotateSpeech(jcas, mainMatter);

		// aggregating annotations
		// TODO: convert to range function
		int currentSceneBegin = -1;
		int currentActBegin = -1;
		for (HTMLAnnotation anno : JCasUtil.select(jcas, HTMLAnnotation.class)) {
			if (anno.getTag().equals("h2") && !anno.getCls().contains("author")) {
				AnnotationFactory.createAnnotation(jcas, anno.getBegin(),
						anno.getEnd(), SceneHeading.class);
				if (currentSceneBegin >= 0) {
					AnnotationFactory.createAnnotation(jcas, currentSceneBegin,
							anno.getBegin() - 1, Scene.class);
				}
				currentSceneBegin = anno.getBegin();
			}
			if (anno.getTag().equals("h1") && !anno.getCls().contains("title")) {
				AnnotationFactory.createAnnotation(jcas, anno.getBegin(),
						anno.getEnd(), ActHeading.class);
				if (currentActBegin >= 0) {
					AnnotationFactory.createAnnotation(jcas, currentActBegin,
							anno.getBegin() - 1, Act.class);
				}
				currentActBegin = anno.getBegin();
			}
		}
		if (currentActBegin >= 0) {
			AnnotationFactory.createAnnotation(jcas, currentActBegin,
					mainMatter.getEnd(), Act.class);
		}
		if (currentSceneBegin >= 0) {
			AnnotationFactory.createAnnotation(jcas, currentSceneBegin,
					mainMatter.getEnd(), Scene.class);
		}
		IMSUtil.trim(new ArrayList<Scene>(JCasUtil.select(jcas, Scene.class)));
		IMSUtil.trim(new ArrayList<Act>(JCasUtil.select(jcas, Act.class)));

		this.cleanUp(jcas);

	}

	protected void annotateSpeech(JCas jcas, Annotation mainMatter) {
		for (Utterance utterance : JCasUtil.selectCovered(Utterance.class,
				mainMatter)) {
			TreeSet<Annotation> except =
					new TreeSet<Annotation>(new Comparator<Annotation>() {

						public int compare(Annotation o1, Annotation o2) {
							return Integer.compare(o1.getBegin(), o2.getBegin());
						}

{'fraction_non_alphanumeric': 0.12430908065425832, 'fraction_numerical': 0.0054145516074450084, 'mean_word_length': 4.96635262449529, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '266218', 'n_tokens_mistral': 3381, 'n_tokens_neox': 3028, 'n_words': 525}