<commit_before>/*
 * jETeL/CloverETL - Java based ETL application framework.
 * Copyright (c) Javlin, a.s. (<EMAIL>)
 *  
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 */
package org.jetel.data.parser;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;

import javax.xml.namespace.QName;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamConstants;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
import org.apache.poi.openxml4j.exceptions.OpenXML4JException;
import org.apache.poi.openxml4j.opc.OPCPackage;
import org.apache.poi.ss.usermodel.BuiltinFormats;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.DateUtil;
import org.apache.poi.xssf.eventusermodel.ReadOnlySharedStringsTable;
import org.apache.poi.xssf.eventusermodel.XSSFReader;
import org.apache.poi.xssf.eventusermodel.XSSFReader.SheetIterator;
import org.apache.poi.xssf.model.StylesTable;
import org.apache.poi.xssf.usermodel.XSSFCellStyle;
import org.jetel.data.DataField;
import org.jetel.data.DataRecord;
import org.jetel.data.parser.SpreadsheetStreamParser.CellBuffers;
import org.jetel.data.parser.SpreadsheetStreamParser.RecordFieldValueSetter;
import org.jetel.data.parser.XSSFSheetXMLHandler.SheetContentsHandler;
import org.jetel.exception.BadDataFormatException;
import org.jetel.exception.ComponentNotReadyException;
import org.jetel.exception.JetelException;
import org.jetel.exception.JetelRuntimeException;
import org.jetel.exception.SpreadsheetException;
import org.jetel.metadata.DataFieldMetadata;
import org.jetel.util.SpreadsheetUtils;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.AttributesImpl;

/**
 * @author tkramolis (<EMAIL>) (c) Javlin, a.s. (www.cloveretl.com)
 * 
 * @created 1 Aug 2011
 */
public class XLSXStreamParser implements SpreadsheetStreamHandler {

	private final static Log LOGGER = LogFactory.getLog(XLSXStreamParser.class);

	private SpreadsheetStreamParser parent;
	
	private String fileName;
	private String sheetName;

	private OPCPackage opcPackage;
	private XSSFReader reader;
	private StylesTable stylesTable;
	private ReadOnlySharedStringsTable sharedStringsTable;
	private XMLStreamReader staxParser;
	private XMLInputFactory xmlInputFactory;
	private RecordFillingContentHandler sheetContentHandler;
	private XSSFSheetXMLHandler xssfContentHandler;

	private SheetIterator sheetIterator;
	private InputStream currentSheetInputStream;

	private CellBuffers<CellValue> cellBuffers;
//	private CellValue[][] cellBuffers;
//	private int nextPartial = -1;

	private int currentSheetIndex;

	private int nextRecordStartRow;

	/** the data formatter used to format cell values as strings */
	private final DataFormatter dataFormatter = new DataFormatter();

	private boolean endOfSheet;
	private int remainingRecordsCount = -1;
	 
	public XLSXStreamParser(SpreadsheetStreamParser parent) {
		this.parent = parent;
	}

	@Override
	public void init() throws ComponentNotReadyException {
		xmlInputFactory = XMLInputFactory.newInstance();
		cellBuffers = parent.new CellBuffers<CellValue>();
	}
	
	@Override
	public DataRecord parseNext(DataRecord record) throws JetelException {
		sheetContentHandler.setRecordStartRow(nextRecordStartRow);
		sheetContentHandler.setRecord(record);
		try {
			while (staxParser.hasNext() && !sheetContentHandler.isRecordFinished()) {
				processParserEvent(staxParser, xssfContentHandler);
			}

			if (!staxParser.hasNext()) {
				endOfSheet = true;
				
				if (sheetContentHandler.isRecordStarted()) {
					if (!sheetContentHandler.isRecordFinished()) {
						// generates exception for record cells beyond end of sheet
						sheetContentHandler.finishRecord();
					}
				} else {
					
					if (remainingRecordsCount < 0) {
						// initialize how many not empty records partially beyond end of sheet are to be returned
						remainingRecordsCount = cellBuffers.getNotEmptyBuffersCount();
						if (remainingRecordsCount != 0 || !record.isNull()) {
							remainingRecordsCount++;
						}
					}
					
					if (remainingRecordsCount == 0) {
						return null;
					}
					
					sheetContentHandler.finishRecord();
				}
			}

		} catch (XMLStreamException e) {
			throw new JetelException("Error occurred while reading XML of sheet " + currentSheetIndex, e);
		} catch (SAXException e) {
			throw new JetelException("Error occurred while reading XML of sheet " + currentSheetIndex, e);
		}

		nextRecordStartRow += parent.mappingInfo.getStep();
		return record;
	}

	@Override
	public int skip(int nRec) throws JetelException {
		if (nRec <= 0) {
			return 0;
		}
		sheetContentHandler.skipRecords(nRec);

		try {
			while (staxParser.hasNext() && sheetContentHandler.isSkipRecords()) {
				processParserEvent(staxParser, xssfContentHandler);
			}

			int skippedRecords = sheetContentHandler.getNumberOfSkippedRecords();
			nextRecordStartRow += skippedRecords * parent.mappingInfo.getStep();<commit_msg>FIX: SpreadsheetReader XLSX stream parser - reading of records partially beyond end of sheet<commit_after>/*
 * jETeL/CloverETL - Java based ETL application framework.
 * Copyright (c) Javlin, a.s. (<EMAIL>)
 *  
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 */
package org.jetel.data.parser;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;

import javax.xml.namespace.QName;
import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamConstants;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
import org.apache.poi.openxml4j.exceptions.OpenXML4JException;
import org.apache.poi.openxml4j.opc.OPCPackage;
import org.apache.poi.ss.usermodel.BuiltinFormats;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.DataFormatter;
import org.apache.poi.ss.usermodel.DateUtil;
import org.apache.poi.xssf.eventusermodel.ReadOnlySharedStringsTable;
import org.apache.poi.xssf.eventusermodel.XSSFReader;
import org.apache.poi.xssf.eventusermodel.XSSFReader.SheetIterator;
import org.apache.poi.xssf.model.StylesTable;
import org.apache.poi.xssf.usermodel.XSSFCellStyle;
import org.jetel.data.DataField;
import org.jetel.data.DataRecord;
import org.jetel.data.parser.SpreadsheetStreamParser.CellBuffers;
import org.jetel.data.parser.SpreadsheetStreamParser.RecordFieldValueSetter;
import org.jetel.data.parser.XSSFSheetXMLHandler.SheetContentsHandler;
import org.jetel.exception.BadDataFormatException;
import org.jetel.exception.ComponentNotReadyException;
import org.jetel.exception.JetelException;
import org.jetel.exception.JetelRuntimeException;
import org.jetel.exception.SpreadsheetException;
import org.jetel.metadata.DataFieldMetadata;
import org.jetel.util.SpreadsheetUtils;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.AttributesImpl;

/**
 * @author tkramolis (<EMAIL>) (c) Javlin, a.s. (www.cloveretl.com)
 * 
 * @created 1 Aug 2011
 */
public class XLSXStreamParser implements SpreadsheetStreamHandler {

	private final static Log LOGGER = LogFactory.getLog(XLSXStreamParser.class);

	private SpreadsheetStreamParser parent;
	
	private String fileName;
	private String sheetName;

	private OPCPackage opcPackage;
	private XSSFReader reader;
	private StylesTable stylesTable;
	private ReadOnlySharedStringsTable sharedStringsTable;
	private XMLStreamReader staxParser;
	private XMLInputFactory xmlInputFactory;
	private RecordFillingContentHandler sheetContentHandler;
	private XSSFSheetXMLHandler xssfContentHandler;

	private SheetIterator sheetIterator;
	private InputStream currentSheetInputStream;

	private CellBuffers<CellValue> cellBuffers;
//	private CellValue[][] cellBuffers;
//	private int nextPartial = -1;

	private int currentSheetIndex;

	private int nextRecordStartRow;

	/** the data formatter used to format cell values as strings */
	private final DataFormatter dataFormatter = new DataFormatter();

	private boolean endOfSheet;
	private int remainingRecordsCount = -1;
	 
	public XLSXStreamParser(SpreadsheetStreamParser parent) {
		this.parent = parent;
	}

	@Override
	public void init() throws ComponentNotReadyException {
		xmlInputFactory = XMLInputFactory.newInstance();
		cellBuffers = parent.new CellBuffers<CellValue>();
	}
	
	@Override
	public DataRecord parseNext(DataRecord record) throws JetelException {
		sheetContentHandler.setRecordStartRow(nextRecordStartRow);
		sheetContentHandler.setRecord(record);
		try {
			while (staxParser.hasNext() && !sheetContentHandler.isRecordFinished()) {
				processParserEvent(staxParser, xssfContentHandler);
			}

			if (!staxParser.hasNext()) {
				endOfSheet = true;
				
				if (sheetContentHandler.isRecordStarted() && remainingRecordsCount < 0) {
					if (!sheetContentHandler.isRecordFinished()) {
						// generates exception for record cells beyond end of sheet
						sheetContentHandler.finishRecord();
					}
				} else {
					
					if (remainingRecordsCount < 0) {
						// initialize how many not empty records partially beyond end of sheet are to be returned
						remainingRecordsCount = cellBuffers.getNotEmptyBuffersCount();
						if (remainingRecordsCount != 0 || !record.isNull()) {
							remainingRecordsCount++;
						}
					}
					
					if (--remainingRecordsCount <= 0) {
						return null;
					}
					
					sheetContentHandler.finishRecord();
				}
			}

		} catch (XMLStreamException e) {
			throw new JetelException("Error occurred while reading XML of sheet " + currentSheetIndex, e);
		} catch (SAXException e) {
			throw new JetelException("Error occurred while reading XML of sheet " + currentSheetIndex, e);
		}

		nextRecordStartRow += parent.mappingInfo.getStep();
		return record;
	}

	@Override
	public int skip(int nRec) throws JetelException {
		if (nRec <= 0) {
			return 0;
		}
		sheetContentHandler.skipRecords(nRec);

		try {
			while (staxParser.hasNext() && sheetContentHandler.isSkipRecords()) {
				processParserEvent(staxParser, xssfContentHandler);
			}

			int skippedRecords = sheetContentHandler.getNumberOfSkippedRecords();
			nextRecordStartRow += skippedRecords * parent.mappingInfo.getStep();
{'fraction_non_alphanumeric': 0.08695290397736728, 'fraction_numerical': 0.005408553835912797, 'mean_word_length': 4.865788189360663, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 24}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1693172', 'n_tokens_mistral': 3905, 'n_tokens_neox': 3494, 'n_words': 905}