<commit_before>package org.openforis.calc.importer;

import java.io.IOException;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.openforis.calc.io.flat.FlatDataStream;
import org.openforis.calc.io.flat.FlatRecord;
import org.springframework.transaction.annotation.Transactional;

/**
 * Not thread-safe, callers must manage synchronization
 * 
 * @author <NAME>
 *
 */
public abstract class AbstractFlatFileImporter {
	private static final int DEFAULT_INSERT_FREQUENCY = 10000;
	private static final int DEFAULT_REPORT_FREQUENCY = 10000;
	private int readRows;
	private int skippedRows;
	private int reportFrequency;
	private int insertFrequency;
	private long startTime;
	private long endTime;
	protected final Log log = LogFactory.getLog(getClass());
	private boolean active;

	public AbstractFlatFileImporter() {
		reportFrequency = DEFAULT_REPORT_FREQUENCY;
		insertFrequency = DEFAULT_INSERT_FREQUENCY;
		startTime = -1;
		endTime = -1;
		active = false;
	}
	
	@Transactional
	public void importData(FlatDataStream stream) throws ImportException, IOException {
		try {
			startTime = System.currentTimeMillis();
			endTime = -1;
			readRows = 0;
			skippedRows = 0;
			active = true;
			
			onStart(stream);

			FlatRecord record;
			while ( (record = stream.nextRecord()) != null ) {
				boolean ok = processRecord(record);
				readRows += 1;
				if ( !ok ) {
					log.warn("Skipping row "+readRows);
					skippedRows += 1;
				}
				if ( getProcessedRows() % insertFrequency == 0 ) {
					performInserts();
				}
				if ( readRows % reportFrequency == 0 ) {
					reportProgress();
				}
			}

			if ( getProcessedRows() % insertFrequency != 0 ) {
				performInserts();
			}
			if ( readRows % reportFrequency != 0 ) {
				reportProgress();
			}
			endTime = System.currentTimeMillis();<commit_msg>Interview importer working<commit_after>package org.openforis.calc.importer;

import java.io.IOException;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.openforis.calc.io.flat.FlatDataStream;
import org.openforis.calc.io.flat.FlatRecord;
import org.springframework.transaction.annotation.Transactional;

/**
 * Not thread-safe, callers must manage synchronization
 * 
 * @author <NAME>
 *
 */
public abstract class AbstractFlatFileImporter {
	private static final int DEFAULT_INSERT_FREQUENCY = 10000;
	private static final int DEFAULT_REPORT_FREQUENCY = 10000;
	private int readRows;
	private int skippedRows;
	private int reportFrequency;
	private int insertFrequency;
	private long startTime;
	private long endTime;
	protected final Log log = LogFactory.getLog(getClass());
	private boolean active;

	public AbstractFlatFileImporter() {
		reportFrequency = DEFAULT_REPORT_FREQUENCY;
		insertFrequency = DEFAULT_INSERT_FREQUENCY;
		startTime = -1;
		endTime = -1;
		active = false;
	}
	
	@Transactional
	public void importData(FlatDataStream stream) throws ImportException, IOException {
		try {
			startTime = System.currentTimeMillis();
			endTime = -1;
			readRows = 0;
			skippedRows = 0;
			active = true;
			
			onStart(stream);

			FlatRecord record;
			while ( (record = stream.nextRecord()) != null ) {
				boolean ok = processRecord(record);
				readRows += 1;
				if ( !ok ) {
					log.warn("Skipping row "+readRows);
					skippedRows += 1;
				}
				if ( getProcessedRows() % insertFrequency == 0 ) {
					performInserts();
				}
				if ( readRows % reportFrequency == 0 ) {
					reportProgress();
				}
			}

			if ( getProcessedRows() % insertFrequency != 0 ) {
				performInserts();
			}
			endTime = System.currentTimeMillis();
{'fraction_non_alphanumeric': 0.10307017543859649, 'fraction_numerical': 0.011239035087719298, 'mean_word_length': 3.9045698924731185, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1569065', 'n_tokens_mistral': 1339, 'n_tokens_neox': 1121, 'n_words': 268}