<commit_before>/*
 * jETeL/CloverETL - Java based ETL application framework.
 * Copyright (c) Javlin, a.s. (<EMAIL>)
 *  
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 */
package org.jetel.data.parser;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.apache.poi.hssf.eventusermodel.FormatTrackingHSSFListener;
import org.apache.poi.hssf.eventusermodel.HSSFEventFactory;
import org.apache.poi.hssf.eventusermodel.HSSFListener;
import org.apache.poi.hssf.eventusermodel.HSSFRequest;
import org.apache.poi.hssf.record.BOFRecord;
import org.apache.poi.hssf.record.BoolErrRecord;
import org.apache.poi.hssf.record.BoundSheetRecord;
import org.apache.poi.hssf.record.CellRecord;
import org.apache.poi.hssf.record.CellValueRecordInterface;
import org.apache.poi.hssf.record.ExtendedFormatRecord;
import org.apache.poi.hssf.record.FormatRecord;
import org.apache.poi.hssf.record.FormulaRecord;
import org.apache.poi.hssf.record.LabelSSTRecord;
import org.apache.poi.hssf.record.MulBlankRecord;
import org.apache.poi.hssf.record.NoteRecord;
import org.apache.poi.hssf.record.NumberRecord;
import org.apache.poi.hssf.record.Record;
import org.apache.poi.hssf.record.RecordFactoryInputStream;
import org.apache.poi.hssf.record.RecordFormatException;
import org.apache.poi.hssf.record.SSTRecord;
import org.apache.poi.hssf.record.SharedFormulaRecord;
import org.apache.poi.hssf.record.StringRecord;
import org.apache.poi.hssf.record.crypto.Biff8EncryptionKey;
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
import org.apache.poi.ss.usermodel.DateUtil;
import org.jetel.data.DataField;
import org.jetel.data.DataRecord;
import org.jetel.data.parser.SpreadsheetStreamParser.CellBuffers;
import org.jetel.data.parser.SpreadsheetStreamParser.RecordFieldValueSetter;
import org.jetel.exception.BadDataFormatException;
import org.jetel.exception.ComponentNotReadyException;
import org.jetel.exception.JetelException;
import org.jetel.exception.JetelRuntimeException;
import org.jetel.metadata.DataFieldMetadata;
import org.jetel.util.SpreadsheetUtils;

/**
 * @author lkrejci (<EMAIL>) (c) Javlin, a.s. (www.cloveretl.com)
 * 
 * @created 22 Aug 2011
 */
public class XLSStreamParser implements SpreadsheetStreamHandler {
	
	private SpreadsheetStreamParser parent;
	private POIFSFileSystem fs;

	private final static Set<Short> ACCEPTED_RECORDS = new HashSet<Short>();
	static {
		ACCEPTED_RECORDS.add(BOFRecord.sid);
		ACCEPTED_RECORDS.add(SSTRecord.sid);
		ACCEPTED_RECORDS.add(NumberRecord.sid);
		ACCEPTED_RECORDS.add(LabelSSTRecord.sid);
		ACCEPTED_RECORDS.add(FormulaRecord.sid);
		ACCEPTED_RECORDS.add(StringRecord.sid);
		ACCEPTED_RECORDS.add(FormulaRecord.sid);
		ACCEPTED_RECORDS.add(EndOfRowRecord.SID);

		ACCEPTED_RECORDS.add(FormatRecord.sid);
		ACCEPTED_RECORDS.add(ExtendedFormatRecord.sid);
	}

	private ParserFinishedFunctionPointer recordReader;
	private ParserFinishedFunctionPointer sheetSetter;
	private ParserFinishedFunctionPointer recordSkipper;

	private MissingRecordAwareFactoryInputStream recordFactory;
	private final RecordFillingHSSFListener recordFillingListener;
	private final FormatTrackingHSSFListener formatter;
	
	private int currentSheetIndex = -1;

	private int nextRecordStartRow;
	
	public XLSStreamParser(SpreadsheetStreamParser parent, String password) {
		this.parent = parent;
		recordFillingListener = new RecordFillingHSSFListener(AbstractSpreadsheetParser.USE_DATE1904);
		formatter = new FormatTrackingHSSFListener(recordFillingListener);
		Biff8EncryptionKey.setCurrentUserPassword(password);
	}

	@Override
	public void init() throws ComponentNotReadyException {
		recordFillingListener.init();
		recordSkipper = new ParserFinishedFunctionPointer() {

			@Override
			public boolean finishedCondition() {
				return !recordFillingListener.skipRecords();
			}

			@Override
			public boolean stopCondition() {
				return !recordFillingListener.isRequestedSheetSet();
			}
		};
		
		recordReader = new ParserFinishedFunctionPointer() {

			@Override
			public boolean finishedCondition() {
				return recordFillingListener.isRecordFinished();
			}

			@Override
			public boolean stopCondition() {
				return !recordFillingListener.isRequestedSheetSet();
			}
		};
		
		sheetSetter = new ParserFinishedFunctionPointer() {

			@Override
			public boolean finishedCondition() {
				// Do not process any more records is requested sheet is set
				return recordFillingListener.isRequestedSheetSet();
			}

			@Override
			public boolean stopCondition() {
				// Never stop processing records for any other reason
				return false;
			}
		};
	}

	@Override
	public DataRecord parseNext(DataRecord record) throws JetelException {
		recordFillingListener.setRecordRange(nextRecordStartRow, parent.mappingInfo.getStep());
		recordFillingListener.setRecord(record);
		if (!processParsing(recordReader)) {
			if (!recordFillingListener.isRecordStarted()) {
				return null;
			}
			recordFillingListener.finishRecord();
		}

		nextRecordStartRow += parent.mappingInfo.getStep();
		return record;
	}

	@Override
	public int skip(int nRec) throws JetelException {
		recordFillingListener.setSkipRecords(nRec);
		processParsing(recordSkipper);

		int skippedRecords = recordFillingListener.getNumberOfSkippedRecords();
		nextRecordStartRow += skippedRecords * parent.mappingInfo.getStep();
		
		return skippedRecords;
	}

	@Override
	public void prepareInput(InputStream inputStream) throws IOException, ComponentNotReadyException {
		fs = new POIFSFileSystem(inputStream);
		prepareRecordFactory();
		currentSheetIndex = -1;
	}
	
	@Override
	public void close() throws IOException {
		Biff8EncryptionKey.setCurrentUserPassword(null);
	}

	private void prepareRecordFactory() {
		try {
			RecordFactoryInputStream rfis = new RecordFactoryInputStream(fs.getRoot().createDocumentInputStream("Workbook"), false);
			recordFactory = new MissingRecordAwareFactoryInputStream(rfis);
		} catch (IOException e) {
			throw new JetelRuntimeException("Failed to open input stream from workbook", e);
		}
	}

	@Override
	public List<String> getSheetNames() throws IOException {
		HSSFEventFactory factory = new HSSFEventFactory();
		HSSFRequest request = new HSSFRequest();

		SheetNamesHSSFListener listener = new SheetNamesHSSFListener();
		request.addListener(listener, BoundSheetRecord.sid);
		factory.processWorkbookEvents(request, fs);

		return listener.getSheetNames();
	}
	
	@Override
	public int getCurrentRecordStartRow() {
		return nextRecordStartRow;
	}

	@Override
	public boolean setCurrentSheet(int sheetNumber) {
		if (currentSheetIndex > sheetNumber) {
			prepareRecordFactory();
		}
		recordFillingListener.setRequestedSheetIndex(sheetNumber);

		if (processParsing(sheetSetter)) {
			this.nextRecordStartRow = parent.startLine;
			return true;
		}

		return false;
	}

	@Override
	public String[][] getHeader(int startRow, int startColumn, int endRow, int endColumn) throws ComponentNotReadyException {
		RecordFactoryInputStream rfis;
		try {
			rfis = new RecordFactoryInputStream(fs.getRoot().createDocumentInputStream("Workbook"), false);
			MissingRecordAwareFactoryInputStream recordFactory = new MissingRecordAwareFactoryInputStream(rfis);
			HeaderHSSFListener headerListener = new HeaderHSSFListener(currentSheetIndex, startRow, startColumn, endRow, endColumn);
			FormatTrackingHSSFListener formatter = new FormatTrackingHSSFListener(headerListener);
			headerListener.setFormatter(formatter);

			Record recordToProcess = null;
			while (!headerListener.isFinished()) {
				recordToProcess = recordFactory.nextRecord();
				if (recordToProcess == null) {
					break;
				}

				if (ACCEPTED_RECORDS.contains(recordToProcess.getSid())) {
					formatter.processRecord(recordToProcess);
				}
			}

			return headerListener.getHeader();
		} catch (IOException e) {
			throw new ComponentNotReadyException(e);
		}
	}

	private static class SheetNamesHSSFListener implements HSSFListener {

		private final List<String> sheetNames = new ArrayList<String>();

		@Override
		public void processRecord(Record record) {
			if (record.getSid() == BoundSheetRecord.sid) {
				sheetNames.add(((BoundSheetRecord) record).getSheetname());
			}
		}

		public List<String> getSheetNames() {
			return sheetNames;
		}
	}

	// TODO: unify somehow with XLSXStreamParser.RecordFillingContentHandler, there is a lot of duplicated code :-/
	//			- some duplicated code already refactored into CellBuffers class
	private class RecordFillingHSSFListener implements HSSFListener, RecordFieldValueSetter<Record> {

		private final boolean date1904;

		/* shared strings table container */
		private SSTRecord sstRecord;
		/* string formula */
		private boolean outputNextStringRecord;
		private int stringFormulaColumnIndex;

		/* skip functionality */
		private int skipStartRow;
		private boolean skipRecords = false;

		/* requested sheet index */
		private int requestedSheetIndex;

		/* record to fill */
		private DataRecord recordToFill;
		/* row on which starts currently read record */
		private int recordStartRow;
		/* row where current record finishes */
		private int recordEndRow;

		/* actual row which is being parsed */
		private int currentParseRow;
		/* flag indicating whether row where current record finishes was read */
		private boolean recordStarted;
		private boolean recordFinished;

		private int lastColumn = -1;

		private CellBuffers<Record> cellBuffers;
		private FieldValueToFormatSetter fieldValueToFormatSetter = new FieldValueToFormatSetter();

		public RecordFillingHSSFListener(boolean date1904) {
			this.date1904 = date1904;
		}

		public void init() {
			currentParseRow = 0;
			cellBuffers = parent.new CellBuffers<Record>();
			cellBuffers.init(Record.class, this, fieldValueToFormatSetter);
		}

		public void setRequestedSheetIndex(int requestedSheetIndex) {
			this.requestedSheetIndex = requestedSheetIndex;

			cellBuffers.clear();
			
			recordStartRow = parent.startLine;
			currentParseRow = 0;
			lastColumn = -1;
		}

		public boolean isRequestedSheetSet() {
			return requestedSheetIndex == currentSheetIndex;
		}

		public void handleMissingCells(int mappingRow, int first, int last) {
			int[] mappingPart = parent.mapping[mappingRow];
			int[] formatsMappingPart = parent.formatMapping != null ? parent.formatMapping[mappingRow] : null;
			for (int i = first + 1; i < last; i++) {
				setFieldToNull(mappingPart, i);
				setFieldToNull(formatsMappingPart, i);
			}
		}

		private void setFieldToNull(int[] mappingArray, int i) {
			if (mappingArray != null) {
				int cloverFieldIndex;
				cloverFieldIndex = mappingArray[i];
				if (cloverFieldIndex != XLSMapping.UNDEFINED) {
					try {
						recordToFill.getField(cloverFieldIndex).setNull(true);
					} catch (BadDataFormatException e) {
						parent.handleException(new BadDataFormatException("There is no data row for field. Moreover, cannot set default value or null", e),
								recordToFill, cloverFieldIndex, null, null);
					}
				}
			}
		}

		public boolean isRecordStarted() {
			return recordStarted;
		}
		
		public boolean isRecordFinished() {
			return recordFinished;
		}
		
		public void finishRecord() {
			for (int i = currentParseRow; i <= recordEndRow; i++) {
				handleMissingCells(i - recordStartRow, lastColumn, parent.mapping[0].length);
			}
		}

		public int getNumberOfSkippedRecords() {
			int result = currentParseRow - skipStartRow + parent.mappingInfo.getStep();
			if (result > 0) {
				return result / parent.mappingInfo.getStep();
			}
			return 0;
		}

		public void setSkipRecords(int nRec) {
			skipStartRow = recordStartRow;
			recordToFill = null;
			
			int numberOfRows = nRec * parent.mappingInfo.getStep();
			setRecordRange(recordStartRow + numberOfRows, parent.mappingInfo.getStep());
			
			if (nRec != 1 || skipStartRow != currentParseRow) {
				skipRecords = true;
			}
		}

		public boolean skipRecords() {
			return skipRecords;
		}

		public void setRecordRange(int startRow, int step) {
			recordStartRow = startRow;
			recordEndRow = startRow + parent.mapping.length - 1;
			recordStarted = false;
			recordFinished = false;
		}

		public void setRecord(DataRecord record) {
			this.recordToFill = record;
			cellBuffers.fillRecordFromBuffer(record);
		}

		@Override
		public void processRecord(Record record) {
			short sid = record.getSid();

			if (sid == SSTRecord.sid) {
				// this record must be read no matter what, as SST stands for shared strings table
				sstRecord = (SSTRecord) record;
				return;
			}

			if (sid == BOFRecord.sid) {
				if (((BOFRecord) record).getType() == BOFRecord.TYPE_WORKSHEET) {
					currentSheetIndex++;

					// Do not skip any more records!
					if (skipRecords) {
						skipRecords = false;
					}
				}
				return;
			}

			if (record instanceof EndOfRowRecord) {
				if (currentSheetIndex != requestedSheetIndex) {
					// Ignore these records
					return;
				}
				if (!skipRecords && currentParseRow >= recordStartRow) {
					handleMissingCells(currentParseRow - recordStartRow, lastColumn, parent.mapping[0].length);
				}
				if (skipRecords && currentParseRow >= recordStartRow - parent.mappingInfo.getStep()) {
					// record skipping is stopped when last row of last record to be skipped is read
					skipRecords = false;
				}

				if (currentParseRow >= recordEndRow) {
					// record reading is finished when last row of record is read<commit_msg>FIX: CL-2083 - SpreadsheetReader XLS stream parser record skipping<commit_after>/*
 * jETeL/CloverETL - Java based ETL application framework.
 * Copyright (c) <NAME>. (<EMAIL>)
 *  
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 */
package org.jetel.data.parser;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;

import org.apache.poi.hssf.eventusermodel.FormatTrackingHSSFListener;
import org.apache.poi.hssf.eventusermodel.HSSFEventFactory;
import org.apache.poi.hssf.eventusermodel.HSSFListener;
import org.apache.poi.hssf.eventusermodel.HSSFRequest;
import org.apache.poi.hssf.record.BOFRecord;
import org.apache.poi.hssf.record.BoolErrRecord;
import org.apache.poi.hssf.record.BoundSheetRecord;
import org.apache.poi.hssf.record.CellRecord;
import org.apache.poi.hssf.record.CellValueRecordInterface;
import org.apache.poi.hssf.record.ExtendedFormatRecord;
import org.apache.poi.hssf.record.FormatRecord;
import org.apache.poi.hssf.record.FormulaRecord;
import org.apache.poi.hssf.record.LabelSSTRecord;
import org.apache.poi.hssf.record.MulBlankRecord;
import org.apache.poi.hssf.record.NoteRecord;
import org.apache.poi.hssf.record.NumberRecord;
import org.apache.poi.hssf.record.Record;
import org.apache.poi.hssf.record.RecordFactoryInputStream;
import org.apache.poi.hssf.record.RecordFormatException;
import org.apache.poi.hssf.record.SSTRecord;
import org.apache.poi.hssf.record.SharedFormulaRecord;
import org.apache.poi.hssf.record.StringRecord;
import org.apache.poi.hssf.record.crypto.Biff8EncryptionKey;
import org.apache.poi.hssf.usermodel.HSSFCell;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
import org.apache.poi.ss.usermodel.DateUtil;
import org.jetel.data.DataField;
import org.jetel.data.DataRecord;
import org.jetel.data.parser.SpreadsheetStreamParser.CellBuffers;
import org.jetel.data.parser.SpreadsheetStreamParser.RecordFieldValueSetter;
import org.jetel.exception.BadDataFormatException;
import org.jetel.exception.ComponentNotReadyException;
import org.jetel.exception.JetelException;
import org.jetel.exception.JetelRuntimeException;
import org.jetel.metadata.DataFieldMetadata;
import org.jetel.util.SpreadsheetUtils;

/**
 * @author lkrejci (<EMAIL>) (c) Javlin, a.s. (www.cloveretl.com)
 * 
 * @created 22 Aug 2011
 */
public class XLSStreamParser implements SpreadsheetStreamHandler {
	
	private SpreadsheetStreamParser parent;
	private POIFSFileSystem fs;

	private final static Set<Short> ACCEPTED_RECORDS = new HashSet<Short>();
	static {
		ACCEPTED_RECORDS.add(BOFRecord.sid);
		ACCEPTED_RECORDS.add(SSTRecord.sid);
		ACCEPTED_RECORDS.add(NumberRecord.sid);
		ACCEPTED_RECORDS.add(LabelSSTRecord.sid);
		ACCEPTED_RECORDS.add(FormulaRecord.sid);
		ACCEPTED_RECORDS.add(StringRecord.sid);
		ACCEPTED_RECORDS.add(FormulaRecord.sid);
		ACCEPTED_RECORDS.add(EndOfRowRecord.SID);

		ACCEPTED_RECORDS.add(FormatRecord.sid);
		ACCEPTED_RECORDS.add(ExtendedFormatRecord.sid);
	}

	private ParserFinishedFunctionPointer recordReader;
	private ParserFinishedFunctionPointer sheetSetter;
	private ParserFinishedFunctionPointer recordSkipper;

	private MissingRecordAwareFactoryInputStream recordFactory;
	private final RecordFillingHSSFListener recordFillingListener;
	private final FormatTrackingHSSFListener formatter;
	
	private int currentSheetIndex = -1;

	private int nextRecordStartRow;
	
	public XLSStreamParser(SpreadsheetStreamParser parent, String password) {
		this.parent = parent;
		recordFillingListener = new RecordFillingHSSFListener(AbstractSpreadsheetParser.USE_DATE1904);
		formatter = new FormatTrackingHSSFListener(recordFillingListener);
		Biff8EncryptionKey.setCurrentUserPassword(password);
	}

	@Override
	public void init() throws ComponentNotReadyException {
		recordFillingListener.init();
		recordSkipper = new ParserFinishedFunctionPointer() {

			@Override
			public boolean finishedCondition() {
				return !recordFillingListener.skipRecords();
			}

			@Override
			public boolean stopCondition() {
				return !recordFillingListener.isRequestedSheetSet();
			}
		};
		
		recordReader = new ParserFinishedFunctionPointer() {

			@Override
			public boolean finishedCondition() {
				return recordFillingListener.isRecordFinished();
			}

			@Override
			public boolean stopCondition() {
				return !recordFillingListener.isRequestedSheetSet();
			}
		};
		
		sheetSetter = new ParserFinishedFunctionPointer() {

			@Override
			public boolean finishedCondition() {
				// Do not process any more records is requested sheet is set
				return recordFillingListener.isRequestedSheetSet();
			}

			@Override
			public boolean stopCondition() {
				// Never stop processing records for any other reason
				return false;
			}
		};
	}

	@Override
	public DataRecord parseNext(DataRecord record) throws JetelException {
		recordFillingListener.setRecordRange(nextRecordStartRow, parent.mappingInfo.getStep());
		recordFillingListener.setRecord(record);
		if (!processParsing(recordReader)) {
			if (!recordFillingListener.isRecordStarted()) {
				return null;
			}
			recordFillingListener.finishRecord();
		}

		nextRecordStartRow += parent.mappingInfo.getStep();
		return record;
	}

	@Override
	public int skip(int nRec) throws JetelException {
		recordFillingListener.setSkipRecords(nRec);
		processParsing(recordSkipper);

		int skippedRecords = recordFillingListener.getNumberOfSkippedRecords();
		nextRecordStartRow += skippedRecords * parent.mappingInfo.getStep();
		
		return skippedRecords;
	}

	@Override
	public void prepareInput(InputStream inputStream) throws IOException, ComponentNotReadyException {
		fs = new POIFSFileSystem(inputStream);
		prepareRecordFactory();
		currentSheetIndex = -1;
	}
	
	@Override
	public void close() throws IOException {
		Biff8EncryptionKey.setCurrentUserPassword(null);
	}

	private void prepareRecordFactory() {
		try {
			RecordFactoryInputStream rfis = new RecordFactoryInputStream(fs.getRoot().createDocumentInputStream("Workbook"), false);
			recordFactory = new MissingRecordAwareFactoryInputStream(rfis);
		} catch (IOException e) {
			throw new JetelRuntimeException("Failed to open input stream from workbook", e);
		}
	}

	@Override
	public List<String> getSheetNames() throws IOException {
		HSSFEventFactory factory = new HSSFEventFactory();
		HSSFRequest request = new HSSFRequest();

		SheetNamesHSSFListener listener = new SheetNamesHSSFListener();
		request.addListener(listener, BoundSheetRecord.sid);
		factory.processWorkbookEvents(request, fs);

		return listener.getSheetNames();
	}
	
	@Override
	public int getCurrentRecordStartRow() {
		return nextRecordStartRow;
	}

	@Override
	public boolean setCurrentSheet(int sheetNumber) {
		if (currentSheetIndex > sheetNumber) {
			prepareRecordFactory();
		}
		recordFillingListener.setRequestedSheetIndex(sheetNumber);

		if (processParsing(sheetSetter)) {
			this.nextRecordStartRow = parent.startLine;
			return true;
		}

		return false;
	}

	@Override
	public String[][] getHeader(int startRow, int startColumn, int endRow, int endColumn) throws ComponentNotReadyException {
		RecordFactoryInputStream rfis;
		try {
			rfis = new RecordFactoryInputStream(fs.getRoot().createDocumentInputStream("Workbook"), false);
			MissingRecordAwareFactoryInputStream recordFactory = new MissingRecordAwareFactoryInputStream(rfis);
			HeaderHSSFListener headerListener = new HeaderHSSFListener(currentSheetIndex, startRow, startColumn, endRow, endColumn);
			FormatTrackingHSSFListener formatter = new FormatTrackingHSSFListener(headerListener);
			headerListener.setFormatter(formatter);

			Record recordToProcess = null;
			while (!headerListener.isFinished()) {
				recordToProcess = recordFactory.nextRecord();
				if (recordToProcess == null) {
					break;
				}

				if (ACCEPTED_RECORDS.contains(recordToProcess.getSid())) {
					formatter.processRecord(recordToProcess);
				}
			}

			return headerListener.getHeader();
		} catch (IOException e) {
			throw new ComponentNotReadyException(e);
		}
	}

	private static class SheetNamesHSSFListener implements HSSFListener {

		private final List<String> sheetNames = new ArrayList<String>();

		@Override
		public void processRecord(Record record) {
			if (record.getSid() == BoundSheetRecord.sid) {
				sheetNames.add(((BoundSheetRecord) record).getSheetname());
			}
		}

		public List<String> getSheetNames() {
			return sheetNames;
		}
	}

	// TODO: unify somehow with XLSXStreamParser.RecordFillingContentHandler, there is a lot of duplicated code :-/
	//			- some duplicated code already refactored into CellBuffers class
	private class RecordFillingHSSFListener implements HSSFListener, RecordFieldValueSetter<Record> {

		private final boolean date1904;

		/* shared strings table container */
		private SSTRecord sstRecord;
		/* string formula */
		private boolean outputNextStringRecord;
		private int stringFormulaColumnIndex;

		/* skip functionality */
		private int skipStartRow;
		private boolean skipRecords = false;

		/* requested sheet index */
		private int requestedSheetIndex;

		/* record to fill */
		private DataRecord recordToFill;
		/* row on which starts currently read record */
		private int recordStartRow;
		/* row where current record finishes */
		private int recordEndRow;

		/* actual row which is being parsed */
		private int currentParseRow;
		/* flag indicating whether row where current record finishes was read */
		private boolean recordStarted;
		private boolean recordFinished;

		private int lastColumn = -1;

		private CellBuffers<Record> cellBuffers;
		private FieldValueToFormatSetter fieldValueToFormatSetter = new FieldValueToFormatSetter();

		public RecordFillingHSSFListener(boolean date1904) {
			this.date1904 = date1904;
		}

		public void init() {
			currentParseRow = 0;
			cellBuffers = parent.new CellBuffers<Record>();
			cellBuffers.init(Record.class, this, fieldValueToFormatSetter);
		}

		public void setRequestedSheetIndex(int requestedSheetIndex) {
			this.requestedSheetIndex = requestedSheetIndex;

			cellBuffers.clear();
			
			recordStartRow = parent.startLine;
			currentParseRow = 0;
			lastColumn = -1;
		}

		public boolean isRequestedSheetSet() {
			return requestedSheetIndex == currentSheetIndex;
		}

		public void handleMissingCells(int mappingRow, int first, int last) {
			int[] mappingPart = parent.mapping[mappingRow];
			int[] formatsMappingPart = parent.formatMapping != null ? parent.formatMapping[mappingRow] : null;
			for (int i = first + 1; i < last; i++) {
				setFieldToNull(mappingPart, i);
				setFieldToNull(formatsMappingPart, i);
			}
		}

		private void setFieldToNull(int[] mappingArray, int i) {
			if (mappingArray != null) {
				int cloverFieldIndex;
				cloverFieldIndex = mappingArray[i];
				if (cloverFieldIndex != XLSMapping.UNDEFINED) {
					try {
						recordToFill.getField(cloverFieldIndex).setNull(true);
					} catch (BadDataFormatException e) {
						parent.handleException(new BadDataFormatException("There is no data row for field. Moreover, cannot set default value or null", e),
								recordToFill, cloverFieldIndex, null, null);
					}
				}
			}
		}

		public boolean isRecordStarted() {
			return recordStarted;
		}
		
		public boolean isRecordFinished() {
			return recordFinished;
		}
		
		public void finishRecord() {
			for (int i = currentParseRow; i <= recordEndRow; i++) {
				handleMissingCells(i - recordStartRow, lastColumn, parent.mapping[0].length);
			}
		}

		public int getNumberOfSkippedRecords() {
			int result = currentParseRow - skipStartRow;
			return result > 0 ? result / parent.mappingInfo.getStep() : 0;
		}

		public void setSkipRecords(int nRec) {
			if (nRec > 0) {
				skipStartRow = recordStartRow;
				recordToFill = null;
				
				int numberOfRows = nRec * parent.mappingInfo.getStep();
				setRecordRange(recordStartRow + numberOfRows, parent.mappingInfo.getStep());
				
				skipRecords = true;
			}
		}

		public boolean skipRecords() {
			return skipRecords;
		}

		public void setRecordRange(int startRow, int step) {
			recordStartRow = startRow;
			recordEndRow = startRow + parent.mapping.length - 1;
			recordStarted = false;
			recordFinished = false;
		}

		public void setRecord(DataRecord record) {
			this.recordToFill = record;
			cellBuffers.fillRecordFromBuffer(record);
		}

		@Override
		public void processRecord(Record record) {
			short sid = record.getSid();

			if (sid == SSTRecord.sid) {
				// this record must be read no matter what, as SST stands for shared strings table
				sstRecord = (SSTRecord) record;
				return;
			}

			if (sid == BOFRecord.sid) {
				if (((BOFRecord) record).getType() == BOFRecord.TYPE_WORKSHEET) {
					currentSheetIndex++;

					// Do not skip any more records! MultiFileReader will continue skipping if too few records were skipped.
					if (skipRecords) {
						skipRecords = false;
					}
				}
				return;
			}

			if (record instanceof EndOfRowRecord) {
				if (currentSheetIndex != requestedSheetIndex) {
					// Ignore these records
					return;
				}
				if (!skipRecords && currentParseRow >= recordStartRow) {
					handleMissingCells(currentParseRow - recordStartRow, lastColumn, parent.mapping[0].length);
				}
				if (skipRecords && currentParseRow >= recordStartRow - 1) {
					// record skipping is stopped when last row of last record to be skipped is read
					skipRecords = false;
				}

				if (currentParseRow >= recordEndRow) {
					// record reading is finished when last row of record is read
{'fraction_non_alphanumeric': 0.0896551724137931, 'fraction_numerical': 0.0042359530894451256, 'mean_word_length': 4.607773851590106, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5011336', 'n_tokens_mistral': 9690, 'n_tokens_neox': 8387, 'n_words': 2024}