<commit_before>package cz.brmlab.yodaqa.analysis.passextract;

import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.lang.Math;

import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence;
import org.apache.uima.UimaContext;
import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
import org.apache.uima.cas.CASException;
import org.apache.uima.fit.component.JCasAnnotator_ImplBase;
import org.apache.uima.fit.descriptor.SofaCapability;
import org.apache.uima.fit.util.JCasUtil;
import org.apache.uima.jcas.JCas;
import org.apache.uima.jcas.tcas.Annotation;
import org.apache.uima.resource.ResourceInitializationException;
import org.apache.uima.util.CasCopier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import cz.brmlab.yodaqa.model.Question.Clue;
import cz.brmlab.yodaqa.model.SearchResult.Passage;
import cz.brmlab.yodaqa.model.SearchResult.ResultInfo;

/**
 * Generate Passages from Sentences that contain some Clue in Question
 * and copy them over to the Passages view.
 *
 * Prospectively, we might want to keep some surrounding sentences,
 * though. */

@SofaCapability(
	inputSofas = { "Question", "Result" },
	outputSofas = { "Passages" }
)


public class PassByClue extends JCasAnnotator_ImplBase {
	final Logger logger = LoggerFactory.getLogger(PassByClue.class);

	public void initialize(UimaContext aContext) throws ResourceInitializationException {
		super.initialize(aContext);
	}

	public void process(JCas jcas) throws AnalysisEngineProcessException {
		JCas questionView, resultView, passagesView;
		try {
			questionView = jcas.getView("Question");
			resultView = jcas.getView("Result");
			passagesView = jcas.getView("Passages");
		} catch (CASException e) {
			throw new AnalysisEngineProcessException(e);
		}
		int totalLength = resultView.getDocumentText().length();

		/* Below, we match clues only as whole words (when we are
		 * looking for "grant", we don't want "vagrant").  However,
		 * this is suboptimal e.g. in case of "nickname" -> "nicknames"
		 * or "nickname" -> "nicknamed", and we didn't do lemmatization
		 * yet. Therefore, as a poor man's fuzzy match, we add an extra
		 * ".?" after the clue to catch simple English lemma forms (and
		 * little in terms of false positives). */

		/* Figure out which clues is the text generally about,
		 * i.e. contained in the title. */
		Map<Clue, Boolean> clueIsAbout = new HashMap<Clue, Boolean>();
		ResultInfo ri = JCasUtil.selectSingle(resultView, ResultInfo.class);
		for (Clue clue : JCasUtil.select(questionView, Clue.class))
			if (ri.getDocumentTitle().matches("(?i).*\\b" + clue.getLabel() + ".?\\b.*"))
				clueIsAbout.put(clue, true);

		/* Pre-index covering info. */
		Map<Sentence, Collection<Annotation>> covering =
			JCasUtil.indexCovered(resultView, Sentence.class, Annotation.class);

		/* Walk sentences, match them and copy what we like. */
		/* XXX: Ideally, we would match all clues in parallel,
		 * but for now we opt for a trivial O(M*N) approach. */
		CasCopier copier = new CasCopier(resultView.getCas(), passagesView.getCas());
		for (Sentence sentence : JCasUtil.select(resultView, Sentence.class)) {
			/* TODO: Put clues in a hierarchy so that we don't
			 * try to match word clues of phrase clues we already
			 * matched. */
			double matches = 0;
			for (Clue clue : JCasUtil.select(questionView, Clue.class)) {
				if (!sentence.getCoveredText().matches("(?i).*\\b" + clue.getLabel() + "(.|ed|ing)?\\b.*"))
					continue;
				/* Match! */<commit_msg>PassByClue: Also allow -ed, -ing suffixes in document title<commit_after>package cz.brmlab.yodaqa.analysis.passextract;

import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.lang.Math;

import de.tudarmstadt.ukp.dkpro.core.api.segmentation.type.Sentence;
import org.apache.uima.UimaContext;
import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
import org.apache.uima.cas.CASException;
import org.apache.uima.fit.component.JCasAnnotator_ImplBase;
import org.apache.uima.fit.descriptor.SofaCapability;
import org.apache.uima.fit.util.JCasUtil;
import org.apache.uima.jcas.JCas;
import org.apache.uima.jcas.tcas.Annotation;
import org.apache.uima.resource.ResourceInitializationException;
import org.apache.uima.util.CasCopier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import cz.brmlab.yodaqa.model.Question.Clue;
import cz.brmlab.yodaqa.model.SearchResult.Passage;
import cz.brmlab.yodaqa.model.SearchResult.ResultInfo;

/**
 * Generate Passages from Sentences that contain some Clue in Question
 * and copy them over to the Passages view.
 *
 * Prospectively, we might want to keep some surrounding sentences,
 * though. */

@SofaCapability(
	inputSofas = { "Question", "Result" },
	outputSofas = { "Passages" }
)


public class PassByClue extends JCasAnnotator_ImplBase {
	final Logger logger = LoggerFactory.getLogger(PassByClue.class);

	public void initialize(UimaContext aContext) throws ResourceInitializationException {
		super.initialize(aContext);
	}

	public void process(JCas jcas) throws AnalysisEngineProcessException {
		JCas questionView, resultView, passagesView;
		try {
			questionView = jcas.getView("Question");
			resultView = jcas.getView("Result");
			passagesView = jcas.getView("Passages");
		} catch (CASException e) {
			throw new AnalysisEngineProcessException(e);
		}
		int totalLength = resultView.getDocumentText().length();

		/* Below, we match clues only as whole words (when we are
		 * looking for "grant", we don't want "vagrant").  However,
		 * this is suboptimal e.g. in case of "nickname" -> "nicknames"
		 * or "nickname" -> "nicknamed", and we didn't do lemmatization
		 * yet. Therefore, as a poor man's fuzzy match, we add an extra
		 * ".?" after the clue to catch simple English lemma forms (and
		 * little in terms of false positives). */

		/* Figure out which clues is the text generally about,
		 * i.e. contained in the title. */
		Map<Clue, Boolean> clueIsAbout = new HashMap<Clue, Boolean>();
		ResultInfo ri = JCasUtil.selectSingle(resultView, ResultInfo.class);
		for (Clue clue : JCasUtil.select(questionView, Clue.class))
			if (ri.getDocumentTitle().matches("(?i).*\\b" + clue.getLabel() + "(.|ed|ing)?\\b.*"))
				clueIsAbout.put(clue, true);

		/* Pre-index covering info. */
		Map<Sentence, Collection<Annotation>> covering =
			JCasUtil.indexCovered(resultView, Sentence.class, Annotation.class);

		/* Walk sentences, match them and copy what we like. */
		/* XXX: Ideally, we would match all clues in parallel,
		 * but for now we opt for a trivial O(M*N) approach. */
		CasCopier copier = new CasCopier(resultView.getCas(), passagesView.getCas());
		for (Sentence sentence : JCasUtil.select(resultView, Sentence.class)) {
			/* TODO: Put clues in a hierarchy so that we don't
			 * try to match word clues of phrase clues we already
			 * matched. */
			double matches = 0;
			for (Clue clue : JCasUtil.select(questionView, Clue.class)) {
				if (!sentence.getCoveredText().matches("(?i).*\\b" + clue.getLabel() + "(.|ed|ing)?\\b.*"))
					continue;
				/* Match! */
{'fraction_non_alphanumeric': 0.12194098727450706, 'fraction_numerical': 0.0008390434904209202, 'mean_word_length': 4.381489841986456, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1463889', 'n_tokens_mistral': 2494, 'n_tokens_neox': 2252, 'n_words': 606}