<commit_before>package org.cloudcoder.dataanalysis;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import org.cloudcoder.app.shared.model.LineCoverage;
import org.cloudcoder.app.shared.model.SubmissionResult;
import org.cloudcoder.app.shared.model.SubmissionResultAnnotation;
import org.cloudcoder.app.shared.model.json.JSONConversion;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.fasterxml.jackson.core.JsonGenerationException;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;

import au.com.bytecode.opencsv.CSV;
import au.com.bytecode.opencsv.CSVWriter;

public class LineCoverageRetestSubmissionResultVisitor implements IRetestSubmissionResultVisitor {
	private static final Logger logger = LoggerFactory.getLogger(LineCoverageRetestSubmissionResultVisitor.class);
	
	private BufferedWriter writer;
	private CSVWriter csvWriter;
	
	@Override
	public void init(File outputDirectory) {
		try {
			writer = new BufferedWriter(new FileWriter(new File(outputDirectory, "coverage.csv")));
			//csvWriter = 
			CSV csv = CSV
					.separator('|')  // delimiter of fields
					.quote('"')      // quote character
					.create();       // new instance is immutable
			csvWriter = csv.writer(writer);

			// We put a heck of a lot of information in each CSV record
			csvWriter.writeNext("submitEventId","fullTextChangeId","courseId","problemId","userId","compilationOutcome","aggregateCoveragePercent","individualCoveragePercent(multi)","aggregateCoverage","individualCoverage(multi)");
		} catch (IOException e) {
			logger.error("Couldn't open coverage results data file", e);
		}
	}

	@Override
	public void onSubmissionResult(SubmissionResult result, RetestSnapshot snapshot) {
		if (writer == null) {
			return;
		}
		
		LineCoverage aggregate = null;
		List<LineCoverage> individual = null;
		String aggregateAsString = null;
		for (SubmissionResultAnnotation annotation : result.getAnnotationList()) {
			String value = annotation.getValue();
			if (annotation.getKey().equals("LineCoverageAggregate")) {
				aggregate = deserializeJson(value);
				aggregateAsString = value;
			} else if (annotation.getKey().equals("LineCoverage")) {
				individual = deserializeJSONArray(value);
			}
		}
		
		if (aggregate != null && individual != null) {
			List<Object> record = new ArrayList<Object>();
			record.addAll(Arrays.asList(
					snapshot.submitEventId,
					snapshot.fullTextChangeId,
					snapshot.courseId,
					snapshot.problemId,
					snapshot.userId,
					result.getCompilationResult().getOutcome().ordinal()
			));
			record.add(aggregate.getPercent());
			for (LineCoverage l : individual) {
				record.add(l.getPercent());
			}
			record.add(aggregateAsString);
			for (LineCoverage l : individual) {
				// The exceptions here shouldn't happen because the LineCoverage
				// objects in the individual list were parsed from JSON, so converting
				// them back to JSON shouldn't fail.
				try {
					record.add(JSONConversion.genericConvertPojoToString(l));
				} catch (JsonGenerationException e) {
					logger.error("Shouldn't happen", e);
				} catch (JsonMappingException e) {
					logger.error("Shouldn't happen", e);
				}
			}
			
			// Write coverage data for this submission result / snapshot
			writeAll(record);
		}
	}

	private LineCoverage deserializeJson(String value) {
		LineCoverage coverage = null;
		try {
			coverage = JSONConversion.genericConvertPojoFromString(value, LineCoverage.class);
		} catch (JsonParseException e) {
			logger.error("Error converting JSON to LineCoverage", e);
		} catch (JsonMappingException e) {
			logger.error("Error converting JSON to LineCoverage", e);
		}
		return coverage;
	}
	
	private List<LineCoverage> deserializeJSONArray(String value) {
		LineCoverage[] coverageList = null;
		try {<commit_msg>ensure coverage info is stored even if partial<commit_after>package org.cloudcoder.dataanalysis;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import org.cloudcoder.app.shared.model.LineCoverage;
import org.cloudcoder.app.shared.model.SubmissionResult;
import org.cloudcoder.app.shared.model.SubmissionResultAnnotation;
import org.cloudcoder.app.shared.model.json.JSONConversion;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.fasterxml.jackson.core.JsonGenerationException;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.databind.JsonMappingException;

import au.com.bytecode.opencsv.CSV;
import au.com.bytecode.opencsv.CSVWriter;

public class LineCoverageRetestSubmissionResultVisitor implements IRetestSubmissionResultVisitor {
	private static final Logger logger = LoggerFactory.getLogger(LineCoverageRetestSubmissionResultVisitor.class);
	
	private BufferedWriter writer;
	private CSVWriter csvWriter;
	
	@Override
	public void init(File outputDirectory) {
		try {
			writer = new BufferedWriter(new FileWriter(new File(outputDirectory, "coverage.csv")));
			//csvWriter = 
			CSV csv = CSV
					.separator('|')  // delimiter of fields
					.quote('"')      // quote character
					.create();       // new instance is immutable
			csvWriter = csv.writer(writer);

			// We put a heck of a lot of information in each CSV record
			csvWriter.writeNext(
				"submitEventId",
				"fullTextChangeId",
				"courseId",
				"problemId",
				"userId",
				"compilationOutcome",
				"aggregateCoveragePercent",
				"aggregateCoverage",
				"individualCoveragePercent(multi)",
				"individualCoverage(multi)"
			);
		} catch (IOException e) {
			logger.error("Couldn't open coverage results data file", e);
		}
	}

	@Override
	public void onSubmissionResult(SubmissionResult result, RetestSnapshot snapshot) {
		if (writer == null) {
			return;
		}
		
		LineCoverage aggregate = null;
		List<LineCoverage> individual = null;
		String aggregateAsString = null;
		for (SubmissionResultAnnotation annotation : result.getAnnotationList()) {
			String value = annotation.getValue();
			if (annotation.getKey().equals("LineCoverageAggregate")) {
				aggregate = deserializeJson(value);
				aggregateAsString = value;
			} else if (annotation.getKey().equals("LineCoverage")) {
				individual = deserializeJSONArray(value);
			}
		}
		
		List<Object> record = new ArrayList<Object>();

		record.addAll(Arrays.asList(
				snapshot.submitEventId,
				snapshot.fullTextChangeId,
				snapshot.courseId,
				snapshot.problemId,
				snapshot.userId,
				result.getCompilationResult().getOutcome().ordinal()
		));

		if (aggregate != null) {
			record.add(aggregate.getPercent());
		}

		if (aggregateAsString != null) {
			record.add(aggregateAsString);
		}

		if (individual != null) {
			for (LineCoverage l : individual) {
				record.add(l.getPercent());
			}

			for (LineCoverage l : individual) {
				// The exceptions here shouldn't happen because the LineCoverage
				// objects in the individual list were parsed from JSON, so converting
				// them back to JSON shouldn't fail.
				try {
					record.add(JSONConversion.genericConvertPojoToString(l));
				} catch (JsonGenerationException e) {
					logger.error("Shouldn't happen", e);
				} catch (JsonMappingException e) {
					logger.error("Shouldn't happen", e);
				}
			}
		}

		// Write coverage data for this submission result / snapshot
		if (!record.isEmpty()) {
			writeAll(record);
		}
	}

	private LineCoverage deserializeJson(String value) {
		LineCoverage coverage = null;
		try {
			coverage = JSONConversion.genericConvertPojoFromString(value, LineCoverage.class);
		} catch (JsonParseException e) {
			logger.error("Error converting JSON to LineCoverage", e);
		} catch (JsonMappingException e) {
			logger.error("Error converting JSON to LineCoverage", e);
		}
		return coverage;
	}
	
	private List<LineCoverage> deserializeJSONArray(String value) {
		LineCoverage[] coverageList = null;
		try {
{'fraction_non_alphanumeric': 0.10755064456721915, 'fraction_numerical': 0.0004910988336402701, 'mean_word_length': 4.3486539724228495, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6082919', 'n_tokens_mistral': 2860, 'n_tokens_neox': 2378, 'n_words': 545}