<commit_before>package at.medunigraz.imi.abbres.dao;

import java.io.Closeable;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Iterator;
import java.util.regex.Pattern;

import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;

import at.medunigraz.imi.abbres.model.Abbreviation;
import at.medunigraz.imi.abbres.model.context.LeftContext;
import at.medunigraz.imi.abbres.model.context.RightContext;

public class ValidationReader implements Closeable, Iterator<Abbreviation> {
	private File file;

	private static final String OUT_OF_SCOPE = "OUT OF SCOPE";
	private static final char ABBREVIATION_MARK = '.';
	private static final String DEFAULT_TOKEN_SEPARATOR = " ";

	private static final Pattern TOKEN_SEPARATOR = Pattern.compile("\\\\");		// This matches a single slash

	private static final int WINDOW_SIZE = 100;

	private XSSFWorkbook workbook;

	private Iterator<Row> rowIterator;

	public ValidationReader(File file) {
		this.file = file;
		openSheet();
	}

	private void openSheet() {
		try {
			workbook = new XSSFWorkbook(new FileInputStream(file));
			XSSFSheet sheet = workbook.getSheetAt(0);
			rowIterator = sheet.iterator();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	@Override
	public void close() throws IOException {
		workbook.close();
	}

	@Override
	public boolean hasNext() {
		return rowIterator.hasNext();
	}

	@Override
	public Abbreviation next() {
		Row row = rowIterator.next();

		Cell cell = row.getCell(1);
		if (cell == null) {
			return null;
		}

		String expansion = cell.getStringCellValue();

		if (expansion.isEmpty()) {
			return null;
		}

		if (expansion.equals(OUT_OF_SCOPE)) {
			return null;
		}

		String sourceText = row.getCell(0).getStringCellValue();

		// Sanity check
		if (sourceText.charAt(WINDOW_SIZE / 2) != ABBREVIATION_MARK) {
			return null;
		}

		// Changes all non-default token separators to a default one.
		sourceText = TOKEN_SEPARATOR.matcher(sourceText).replaceAll(DEFAULT_TOKEN_SEPARATOR);

		int lastSeparator = sourceText.lastIndexOf(DEFAULT_TOKEN_SEPARATOR, WINDOW_SIZE / 2);
		String token = sourceText.substring(lastSeparator + 1, WINDOW_SIZE / 2 + 1);

		String leftString = sourceText.substring(0, lastSeparator).trim();
		LeftContext leftContext = new LeftContext(leftString);

		String rightString = sourceText.substring(WINDOW_SIZE / 2 + 1, sourceText.length()).trim();
		RightContext rightContext = new RightContext(rightString);

		return new Abbreviation(token).withExpansion(expansion).withLeftContext(leftContext)
				.withRightContext(rightContext);
	}
}<commit_msg>Adds debug messages where expected conditions did not happen<commit_after>package at.medunigraz.imi.abbres.dao;

import java.io.Closeable;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.Iterator;
import java.util.regex.Pattern;

import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.xssf.usermodel.XSSFSheet;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import at.medunigraz.imi.abbres.model.Abbreviation;
import at.medunigraz.imi.abbres.model.context.LeftContext;
import at.medunigraz.imi.abbres.model.context.RightContext;
import at.medunigraz.imi.abbres.stats.Evaluator;

public class ValidationReader implements Closeable, Iterator<Abbreviation> {
	private static final Logger LOG = LoggerFactory.getLogger(Evaluator.class);

	private File file;

	private static final String OUT_OF_SCOPE = "OUT OF SCOPE";
	private static final char ABBREVIATION_MARK = '.';
	private static final String DEFAULT_TOKEN_SEPARATOR = " ";

	// This matches a single slash
	private static final Pattern TOKEN_SEPARATOR = Pattern.compile("\\\\");

	private static final int WINDOW_SIZE = 100;

	private XSSFWorkbook workbook;

	private Iterator<Row> rowIterator;

	public ValidationReader(File file) {
		this.file = file;
		openSheet();
	}

	private void openSheet() {
		try {
			workbook = new XSSFWorkbook(new FileInputStream(file));
			XSSFSheet sheet = workbook.getSheetAt(0);
			rowIterator = sheet.iterator();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	@Override
	public void close() throws IOException {
		workbook.close();
	}

	@Override
	public boolean hasNext() {
		return rowIterator.hasNext();
	}

	@Override
	public Abbreviation next() {
		Row row = rowIterator.next();

		Cell cell = row.getCell(1);
		if (cell == null) {
			return null;
		}

		String expansion = cell.getStringCellValue();

		if (expansion.isEmpty()) {
			LOG.debug("Empty expansion at row number " + row.getRowNum());
			return null;
		}

		if (expansion.equals(OUT_OF_SCOPE)) {
			return null;
		}

		String sourceText = row.getCell(0).getStringCellValue();

		// Sanity check
		if (sourceText.charAt(WINDOW_SIZE / 2) != ABBREVIATION_MARK) {
			LOG.debug("Sanity check did not pass at row number " + row.getRowNum());
			return null;
		}

		// Changes all non-default token separators to a default one.
		sourceText = TOKEN_SEPARATOR.matcher(sourceText).replaceAll(DEFAULT_TOKEN_SEPARATOR);

		int lastSeparator = sourceText.lastIndexOf(DEFAULT_TOKEN_SEPARATOR, WINDOW_SIZE / 2);
		String token = sourceText.substring(lastSeparator + 1, WINDOW_SIZE / 2 + 1);

		String leftString = sourceText.substring(0, lastSeparator).trim();
		LeftContext leftContext = new LeftContext(leftString);

		String rightString = sourceText.substring(WINDOW_SIZE / 2 + 1, sourceText.length()).trim();
		RightContext rightContext = new RightContext(rightString);

		return new Abbreviation(token).withExpansion(expansion).withLeftContext(leftContext)
				.withRightContext(rightContext);
	}
}
{'fraction_non_alphanumeric': 0.10946291560102302, 'fraction_numerical': 0.005115089514066497, 'mean_word_length': 4.673114119922631, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '15367', 'n_tokens_mistral': 2080, 'n_tokens_neox': 1891, 'n_words': 427}