<commit_before>package cz.brmlab.yodaqa.io.debug;

import java.util.Iterator;
import java.util.regex.Pattern;
import java.io.IOException;
import java.io.PrintWriter;
import java.lang.Math;

import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.uima.UimaContext;
import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
import org.apache.uima.cas.FSIndex;
import org.apache.uima.cas.FSIterator;
import org.apache.uima.fit.component.JCasConsumer_ImplBase;
import org.apache.uima.fit.descriptor.ConfigurationParameter;
import org.apache.uima.fit.util.JCasUtil;
import org.apache.uima.jcas.JCas;
import org.apache.uima.resource.ResourceInitializationException;

import cz.brmlab.yodaqa.flow.dashboard.Question;
import cz.brmlab.yodaqa.flow.dashboard.QuestionDashboard;
import cz.brmlab.yodaqa.model.AnswerHitlist.Answer;
import cz.brmlab.yodaqa.model.Question.Concept;
import cz.brmlab.yodaqa.model.Question.QuestionInfo;
import cz.brmlab.yodaqa.model.Question.SV;
import cz.brmlab.yodaqa.model.TyCor.LAT;

/**
 * A consumer that displays the questions with a LAT dump to json file
 * The format is:
 * {"qId": "...", "SV": ["...", "..."], "LAT" : [ {"synset" : "...", "text" : "...", "specificity" : "..." "type" : "..."}, {...}, {...}]} \n
 * Pair this with TSVQuestionReader or JSONQuestionReader e.g. on data/eval/.
 */

public class QuestionPrinter extends JCasConsumer_ImplBase {

	public static final String PARAM_JSONFILE = "JSONFILE";
	@ConfigurationParameter(name = PARAM_JSONFILE, mandatory = true)
		private String JSONFile;
	PrintWriter JSONOutput;


	public synchronized void initialize(UimaContext context)
		throws ResourceInitializationException {
		super.initialize(context);

		try {
			JSONOutput = new PrintWriter(JSONFile);
		} catch (IOException io) {
			throw new ResourceInitializationException(io);
		}
	}

	protected void output(String res)
	{
		//System.out.println(res);
		JSONOutput.println(res);
		JSONOutput.flush();
	}

	public synchronized void process(JCas jcas) throws AnalysisEngineProcessException {
		JCas questionView;
		try {
			questionView = jcas;
		} catch (Exception e) {
			throw new AnalysisEngineProcessException(e);
		}

		QuestionInfo qi = JCasUtil.selectSingle(questionView, QuestionInfo.class);
		/*{"qId": "...", "sv": "...", "LAT" : [ {...}, {...}, {...}]} */
		String line = "{\"qId\": " + "\"" + qi.getQuestionId() + "\"" + ", " + "\"SV\": ";

		String SVtmp ="[";
		for (Iterator SVIterator = JCasUtil.select(jcas, SV.class).iterator(); SVIterator.hasNext(); ) {
			SV sv = (SV) SVIterator.next();
			SVtmp += "\"" + sv.getCoveredText() + "\"";
			if(SVIterator.hasNext()){
				SVtmp += ", ";
			}
		}
		SVtmp += "], ";
		line += SVtmp;

		line += "\"LAT\": ";
		String LATtmp = "[";
		for (Iterator iterator = JCasUtil.select(jcas, LAT.class).iterator(); iterator.hasNext(); ) {
			LAT l = (LAT) iterator.next();
			/*{"synset" : "...", "text" : "...", "specificity" : "..." "type" : "..."}*/
			LATtmp += "{";
			if (l.getSynset() != 0) { //only add synset when it is not zero
				LATtmp += "\"synset\": " + "\"" + l.getSynset() + "\", ";
			}
			//add the rest
			LATtmp += "\"text\": \"" + l.getText() + "\"," + " \"specificity\": \"" + l.getSpecificity() + "\", " + "\"type\": " +
				"\"" + l.getClass().getSimpleName() + "\"}";
			//not last, add comma
			if (iterator.hasNext()) {
				LATtmp += ", ";
			}
		}
		LATtmp += "], ";
		line += LATtmp;

		line += "\"Concept\": ";
		String Concepttmp = "[";
		for (Iterator iterator = JCasUtil.select(jcas, Concept.class).iterator(); iterator.hasNext(); ) {
			Concept c = (Concept) iterator.next();
			Concepttmp += "{";
			Concepttmp += "\"fullLabel\": \"" + c.getFullLabel().replaceAll("\"", "\\\"") + "\", ";
			Concepttmp += "\"cookedLabel\": \"" + c.getCookedLabel().replaceAll("\"", "\\\"") + "\", ";
			Concepttmp += "\"pageID\": \"" + c.getPageID() + "\"";
			Concepttmp += "}";
			//not last, add comma
			if (iterator.hasNext()) {
				Concepttmp += ", ";
			}
		}
		Concepttmp += "], ";
		line += Concepttmp;

		line += "}";
		output(line);
		//Question q = QuestionDashboard.getInstance().get(qi.getQuestionId());
		//QuestionDashboard.getInstance().finishQuestion(q);
	}
}<commit_msg>Reparied escaping of double quotes<commit_after>package cz.brmlab.yodaqa.io.debug;

import java.util.Iterator;
import java.util.regex.Pattern;
import java.io.IOException;
import java.io.PrintWriter;
import java.lang.Math;

import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.uima.UimaContext;
import org.apache.uima.analysis_engine.AnalysisEngineProcessException;
import org.apache.uima.cas.FSIndex;
import org.apache.uima.cas.FSIterator;
import org.apache.uima.fit.component.JCasConsumer_ImplBase;
import org.apache.uima.fit.descriptor.ConfigurationParameter;
import org.apache.uima.fit.util.JCasUtil;
import org.apache.uima.jcas.JCas;
import org.apache.uima.resource.ResourceInitializationException;

import cz.brmlab.yodaqa.flow.dashboard.Question;
import cz.brmlab.yodaqa.flow.dashboard.QuestionDashboard;
import cz.brmlab.yodaqa.model.AnswerHitlist.Answer;
import cz.brmlab.yodaqa.model.Question.Concept;
import cz.brmlab.yodaqa.model.Question.QuestionInfo;
import cz.brmlab.yodaqa.model.Question.SV;
import cz.brmlab.yodaqa.model.TyCor.LAT;

/**
 * A consumer that displays the questions with a LAT dump to json file
 * The format is:
 * {"qId": "...", "SV": ["...", "..."], "LAT" : [ {"synset" : "...", "text" : "...", "specificity" : "..." "type" : "..."}, {...}, {...}]} \n
 * Pair this with TSVQuestionReader or JSONQuestionReader e.g. on data/eval/.
 */

public class QuestionPrinter extends JCasConsumer_ImplBase {

	public static final String PARAM_JSONFILE = "JSONFILE";
	@ConfigurationParameter(name = PARAM_JSONFILE, mandatory = true)
		private String JSONFile;
	PrintWriter JSONOutput;


	public synchronized void initialize(UimaContext context)
		throws ResourceInitializationException {
		super.initialize(context);

		try {
			JSONOutput = new PrintWriter(JSONFile);
		} catch (IOException io) {
			throw new ResourceInitializationException(io);
		}
	}

	protected void output(String res)
	{
		//System.out.println(res);
		JSONOutput.println(res);
		JSONOutput.flush();
	}

	public synchronized void process(JCas jcas) throws AnalysisEngineProcessException {
		JCas questionView;
		try {
			questionView = jcas;
		} catch (Exception e) {
			throw new AnalysisEngineProcessException(e);
		}

		QuestionInfo qi = JCasUtil.selectSingle(questionView, QuestionInfo.class);
		/*{"qId": "...", "sv": "...", "LAT" : [ {...}, {...}, {...}]} */
		String line = "{\"qId\": " + "\"" + qi.getQuestionId() + "\"" + ", " + "\"SV\": ";

		String SVtmp ="[";
		for (Iterator SVIterator = JCasUtil.select(jcas, SV.class).iterator(); SVIterator.hasNext(); ) {
			SV sv = (SV) SVIterator.next();
			SVtmp += "\"" + sv.getCoveredText() + "\"";
			if(SVIterator.hasNext()){
				SVtmp += ", ";
			}
		}
		SVtmp += "], ";
		line += SVtmp;

		line += "\"LAT\": ";
		String LATtmp = "[";
		for (Iterator iterator = JCasUtil.select(jcas, LAT.class).iterator(); iterator.hasNext(); ) {
			LAT l = (LAT) iterator.next();
			/*{"synset" : "...", "text" : "...", "specificity" : "..." "type" : "..."}*/
			LATtmp += "{";
			if (l.getSynset() != 0) { //only add synset when it is not zero
				LATtmp += "\"synset\": " + "\"" + l.getSynset() + "\", ";
			}
			//add the rest
			LATtmp += "\"text\": \"" + l.getText() + "\"," + " \"specificity\": \"" + l.getSpecificity() + "\", " + "\"type\": " +
				"\"" + l.getClass().getSimpleName() + "\"}";
			//not last, add comma
			if (iterator.hasNext()) {
				LATtmp += ", ";
			}
		}
		LATtmp += "], ";
		line += LATtmp;

		line += "\"Concept\": ";
		String Concepttmp = "[";
		for (Iterator iterator = JCasUtil.select(jcas, Concept.class).iterator(); iterator.hasNext(); ) {
			Concept c = (Concept) iterator.next();
			Concepttmp += "{";
			Concepttmp += "\"fullLabel\": \"" + c.getFullLabel().replaceAll("\"", "\\\\\"") + "\", ";
			Concepttmp += "\"cookedLabel\": \"" + c.getCookedLabel().replaceAll("\"", "\\\\\"") + "\", ";
			Concepttmp += "\"pageID\": \"" + c.getPageID() + "\"";
			Concepttmp += "}";
			//not last, add comma
			if (iterator.hasNext()) {
				Concepttmp += ", ";
			}
		}
		Concepttmp += "], ";
		line += Concepttmp;

		line += "}";
		output(line);
		//Question q = QuestionDashboard.getInstance().get(qi.getQuestionId());
		//QuestionDashboard.getInstance().finishQuestion(q);
	}
}
{'fraction_non_alphanumeric': 0.20329477742726954, 'fraction_numerical': 0.00023367215796237878, 'mean_word_length': 3.6046261430876814, 'pattern_counts': {'":': 30, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1691705', 'n_tokens_mistral': 3224, 'n_tokens_neox': 2937, 'n_words': 542}