<commit_before>package de.unistuttgart.quadrama.io.tei.textgrid;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map;

import org.apache.commons.io.IOUtils;
import org.apache.uima.collection.CollectionException;
import org.apache.uima.fit.factory.AnnotationFactory;
import org.apache.uima.fit.util.JCasUtil;
import org.apache.uima.jcas.JCas;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.parser.Parser;

import de.unistuttgart.ims.uimautil.IMSUtil;
import de.unistuttgart.quadrama.api.Act;
import de.unistuttgart.quadrama.api.Drama;
import de.unistuttgart.quadrama.api.DramatisPersonae;
import de.unistuttgart.quadrama.api.Figure;
import de.unistuttgart.quadrama.api.FigureDescription;
import de.unistuttgart.quadrama.api.FrontMatter;
import de.unistuttgart.quadrama.api.MainMatter;
import de.unistuttgart.quadrama.api.Scene;
import de.unistuttgart.quadrama.api.Speaker;
import de.unistuttgart.quadrama.api.Speech;
import de.unistuttgart.quadrama.api.StageDirection;
import de.unistuttgart.quadrama.api.Utterance;
import de.unistuttgart.quadrama.io.core.AbstractDramaReader;
import de.unistuttgart.quadrama.io.core.type.HTMLAnnotation;

public class TextgridTEIReader extends AbstractDramaReader {

	@Override
	public void getNext(JCas jcas) throws IOException, CollectionException {
		File file = files[current++];
		Drama drama = new Drama(jcas);
		drama.addToIndexes();
		drama.setDocumentId(file.getName());

		jcas.setDocumentLanguage(language);

		String str = IOUtils.toString(new FileInputStream(file));
		Document doc = Jsoup.parse(str, "", Parser.xmlParser());

		Visitor vis = new Visitor(jcas);

		Element root = doc.select("TEI > text").first();
		root.traverse(vis);
		jcas = vis.getJCas();

		select2Annotation(jcas, root, vis.getAnnotationMap(), "front",
				FrontMatter.class, null);
		select2Annotation(jcas, root, vis.getAnnotationMap(), "body",
				MainMatter.class, null);

		MainMatter mainMatter = JCasUtil.selectSingle(jcas, MainMatter.class);

		select2Annotation(jcas, root, vis.getAnnotationMap(), "speaker",
				Speaker.class, null);
		select2Annotation(jcas, root, vis.getAnnotationMap(), "stage",
				StageDirection.class, mainMatter);
		select2Annotation(jcas, root, vis.getAnnotationMap(), "sp",
				Utterance.class, null);
		select2Annotation(jcas, root, vis.getAnnotationMap(), "l",
				Speech.class, mainMatter);
		select2Annotation(jcas, root, vis.getAnnotationMap(), "ab",
				Speech.class, mainMatter);
		select2Annotation(jcas, root, vis.getAnnotationMap(), "p",
				Speech.class, mainMatter);

		readActsAndScenes(jcas, root, vis.getAnnotationMap());
		readDramatisPersonae(jcas, root, vis.getAnnotationMap());

		fixSpeakerAnnotations(jcas);

		cleanUp(jcas);

	}

	private void readActsAndScenes(JCas jcas, Element root,
			Map<String, HTMLAnnotation> map) {

		if (!root.select("div[type=scene]").isEmpty()) {
			select2Annotation(jcas, root, map, "div[type=scene]", Scene.class,
					null);
		}
		if (!root.select("div[type=act]").isEmpty()) {
			select2Annotation(jcas, root, map, "div[type=act]", Act.class, null);
		}
		if (!JCasUtil.exists(jcas, Act.class)) {
			select2Annotation(jcas, root, map, "body > div", Act.class, null);
		}
		if (!JCasUtil.exists(jcas, Scene.class)) {
			select2Annotation(jcas, root, map, "div > div", Scene.class, null);
		}
	}

	private void readDramatisPersonae(JCas jcas, Element root,<commit_msg>trying to detect headings for acts and scenes<commit_after>package de.unistuttgart.quadrama.io.tei.textgrid;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Collection;
import java.util.HashSet;
import java.util.Map;

import org.apache.commons.io.IOUtils;
import org.apache.uima.collection.CollectionException;
import org.apache.uima.fit.factory.AnnotationFactory;
import org.apache.uima.fit.util.JCasUtil;
import org.apache.uima.jcas.JCas;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.parser.Parser;

import de.unistuttgart.ims.uimautil.IMSUtil;
import de.unistuttgart.quadrama.api.Act;
import de.unistuttgart.quadrama.api.ActHeading;
import de.unistuttgart.quadrama.api.Drama;
import de.unistuttgart.quadrama.api.DramatisPersonae;
import de.unistuttgart.quadrama.api.Figure;
import de.unistuttgart.quadrama.api.FigureDescription;
import de.unistuttgart.quadrama.api.FrontMatter;
import de.unistuttgart.quadrama.api.MainMatter;
import de.unistuttgart.quadrama.api.Scene;
import de.unistuttgart.quadrama.api.SceneHeading;
import de.unistuttgart.quadrama.api.Speaker;
import de.unistuttgart.quadrama.api.Speech;
import de.unistuttgart.quadrama.api.StageDirection;
import de.unistuttgart.quadrama.api.Utterance;
import de.unistuttgart.quadrama.io.core.AbstractDramaReader;
import de.unistuttgart.quadrama.io.core.type.HTMLAnnotation;

public class TextgridTEIReader extends AbstractDramaReader {

	@Override
	public void getNext(JCas jcas) throws IOException, CollectionException {
		File file = files[current++];
		Drama drama = new Drama(jcas);
		drama.addToIndexes();
		drama.setDocumentId(file.getName());

		jcas.setDocumentLanguage(language);

		String str = IOUtils.toString(new FileInputStream(file));
		Document doc = Jsoup.parse(str, "", Parser.xmlParser());

		Visitor vis = new Visitor(jcas);

		Element root = doc.select("TEI > text").first();
		root.traverse(vis);
		jcas = vis.getJCas();

		select2Annotation(jcas, root, vis.getAnnotationMap(), "front",
				FrontMatter.class, null);
		select2Annotation(jcas, root, vis.getAnnotationMap(), "body",
				MainMatter.class, null);

		MainMatter mainMatter = JCasUtil.selectSingle(jcas, MainMatter.class);

		select2Annotation(jcas, root, vis.getAnnotationMap(), "speaker",
				Speaker.class, null);
		select2Annotation(jcas, root, vis.getAnnotationMap(), "stage",
				StageDirection.class, mainMatter);
		select2Annotation(jcas, root, vis.getAnnotationMap(), "sp",
				Utterance.class, null);
		select2Annotation(jcas, root, vis.getAnnotationMap(), "l",
				Speech.class, mainMatter);
		select2Annotation(jcas, root, vis.getAnnotationMap(), "ab",
				Speech.class, mainMatter);
		select2Annotation(jcas, root, vis.getAnnotationMap(), "p",
				Speech.class, mainMatter);

		readActsAndScenes(jcas, root, vis.getAnnotationMap());
		readDramatisPersonae(jcas, root, vis.getAnnotationMap());

		fixSpeakerAnnotations(jcas);

		cleanUp(jcas);

	}

	private void readActsAndScenes(JCas jcas, Element root,
			Map<String, HTMLAnnotation> map) {

		if (!root.select("div[type=scene]").isEmpty()) {
			select2Annotation(jcas, root, map, "div[type=scene]", Scene.class,
					null);
			select2Annotation(jcas, root, map,
					"div[type=scene] > div > desc > title", SceneHeading.class,
					null);
		}
		if (!root.select("div[type=act]").isEmpty()) {
			select2Annotation(jcas, root, map, "div[type=act]", Act.class, null);
			select2Annotation(jcas, root, map,
					"div[type=act] > div > desc > title", ActHeading.class,
					null);
		}
		if (!JCasUtil.exists(jcas, Act.class)) {
			select2Annotation(jcas, root, map, "body > div", Act.class, null);
		}
		if (!JCasUtil.exists(jcas, Scene.class)) {
			select2Annotation(jcas, root, map, "div > div", Scene.class, null);
		}
	}

	private void readDramatisPersonae(JCas jcas, Element root,
{'fraction_non_alphanumeric': 0.1346412254770223, 'fraction_numerical': 0.0034936844934157483, 'mean_word_length': 5.506118881118881, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1800152', 'n_tokens_mistral': 2801, 'n_tokens_neox': 2599, 'n_words': 458}