<commit_before>/*
*    jETeL/Clover - Java based ETL application framework.
*    Copyright (C) 2002-04  <NAME> <<EMAIL>>
*    
*    This library is free software; you can redistribute it and/or
*    modify it under the terms of the GNU Lesser General Public
*    License as published by the Free Software Foundation; either
*    version 2.1 of the License, or (at your option) any later version.
*    
*    This library is distributed in the hope that it will be useful,
*    but WITHOUT ANY WARRANTY; without even the implied warranty of
*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU    
*    Lesser General Public License for more details.
*    
*    You should have received a copy of the GNU Lesser General Public
*    License along with this library; if not, write to the Free Software
*    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*
*/
// FILE: c:/projects/jetel/org/jetel/data/FixLenDataParser.java

package org.jetel.data.parser;
import java.nio.*;
import java.nio.channels.*;
import java.nio.charset.*;
import java.io.*;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.jetel.data.DataRecord;
import org.jetel.data.Defaults;
import org.jetel.exception.BadDataFormatException;
import org.jetel.exception.BadDataFormatExceptionHandler;
import org.jetel.exception.JetelException;
import org.jetel.metadata.*;

/**
 *  Parsing fix length data.  It should be used in cases where new lines 
 *  do not separate records.  Supports fields up to <tt><b>FIELD_BUFFER_LENGTH</b></tt> long.
 *  Size of each individual field must be specified - through metadata definition.
 *
 *  This class is using the new IO (NIO) features introduced in Java 1.4 - directly mapped
 *  byte buffers & character encoders/decoders.
 * 
 * @author     <NAME>
 * @since    August 21, 2002
 * @see        Parser
 * @see      Defaults
 * @revision    $Revision$
 */
public class FixLenDataParser implements Parser {

	private BadDataFormatExceptionHandler handlerBDFE;
	private ByteBuffer dataBuffer;
	private ByteBuffer fieldBuffer;
	//private CharBuffer fieldStringBuffer;  //not used
	private DataRecordMetadata metadata;

	private ReadableByteChannel reader;
	private CharsetDecoder decoder;
	private int recordCounter;
	private int fieldLengths[];

	static Log logger = LogFactory.getLog(FixLenDataParser.class);
	
	// Attributes

	/**
	 *Constructor for the FixLenDataParser object
	 *
	 * @since    August 21, 2002
	 */
	public FixLenDataParser() {
		dataBuffer = ByteBuffer.allocateDirect(Defaults.DEFAULT_INTERNAL_IO_BUFFER_SIZE);

		fieldBuffer = ByteBuffer.allocateDirect(Defaults.DataParser.FIELD_BUFFER_LENGTH);
		decoder = Charset.forName(Defaults.DataParser.DEFAULT_CHARSET_DECODER).newDecoder();
	}


	/**
	 *Constructor for the FixLenDataParser object
	 *
	 * @param  charsetDecoder  Description of Parameter
	 * @since                  August 21, 2002
	 */
	public FixLenDataParser(String charsetDecoder) {
		dataBuffer = ByteBuffer.allocateDirect(Defaults.DEFAULT_INTERNAL_IO_BUFFER_SIZE);

		fieldBuffer = ByteBuffer.allocateDirect(Defaults.DataParser.FIELD_BUFFER_LENGTH);
		decoder = Charset.forName(charsetDecoder).newDecoder();
	}


	/**
	 *  Gets the Next attribute of the FixLenDataParser object
	 *
	 * @return                  The Next value
	 * @exception  IOException  Description of Exception
	 * @since                   August 21, 2002
	 */
	public DataRecord getNext() throws JetelException {
		// create a new data record
		DataRecord record = new DataRecord(metadata);

		record.init();

		record = parseNext(record);
		if(handlerBDFE != null ) {  //use handler only if configured
			while(handlerBDFE.isThrowException()) {
				handlerBDFE.handleException(record);
				record = parseNext(record);
			}
		}
		return record;
	}


	/**
	 *  Gets the Next attribute of the FixLenDataParser object
	 *
	 * @param  record           Description of Parameter
	 * @return                  The Next value
	 * @exception  IOException  Description of Exception
	 * @since                   August 21, 2002
	 */
	public DataRecord getNext(DataRecord record) throws JetelException {
		record = parseNext(record);
		if(handlerBDFE != null ) {  //use handler only if configured
			while(handlerBDFE.isThrowException()) {
				handlerBDFE.handleException(record);
				record = parseNext(record);
			}
		}
		return record;
	}


	/**
	 *  Description of the Method
	 *
	 * @param  in         Description of Parameter
	 * @param  _metadata  Description of Parameter
	 * @since             August 21, 2002
	 */
	public void open(Object in, DataRecordMetadata _metadata) {
		CoderResult result;
		this.metadata = _metadata;

		reader = ((FileInputStream) in).getChannel();

		// create array of field sizes & initialize them
		fieldLengths = new int[metadata.getNumFields()];
		for (int i = 0; i < metadata.getNumFields(); i++) {
			fieldLengths[i] = metadata.getField(i).getSize();
		}
		decoder.reset();
		// reset CharsetDecoder
		recordCounter = 1;
		// reset record counter
		dataBuffer.clear();
		dataBuffer.compact();
	}
<commit_msg>"open" metod for each kind of InputStream<commit_after>/*
*    jETeL/Clover - Java based ETL application framework.
*    Copyright (C) 2002-04  <NAME> <<EMAIL>>
*    
*    This library is free software; you can redistribute it and/or
*    modify it under the terms of the GNU Lesser General Public
*    License as published by the Free Software Foundation; either
*    version 2.1 of the License, or (at your option) any later version.
*    
*    This library is distributed in the hope that it will be useful,
*    but WITHOUT ANY WARRANTY; without even the implied warranty of
*    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU    
*    Lesser General Public License for more details.
*    
*    You should have received a copy of the GNU Lesser General Public
*    License along with this library; if not, write to the Free Software
*    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
*
*/
// FILE: c:/projects/jetel/org/jetel/data/FixLenDataParser.java

package org.jetel.data.parser;
import java.nio.*;
import java.nio.channels.*;
import java.nio.charset.*;
import java.io.*;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.jetel.data.DataRecord;
import org.jetel.data.Defaults;
import org.jetel.exception.BadDataFormatException;
import org.jetel.exception.BadDataFormatExceptionHandler;
import org.jetel.exception.JetelException;
import org.jetel.metadata.*;

/**
 *  Parsing fix length data.  It should be used in cases where new lines 
 *  do not separate records.  Supports fields up to <tt><b>FIELD_BUFFER_LENGTH</b></tt> long.
 *  Size of each individual field must be specified - through metadata definition.
 *
 *  This class is using the new IO (NIO) features introduced in Java 1.4 - directly mapped
 *  byte buffers & character encoders/decoders.
 * 
 * @author     <NAME>
 * @since    August 21, 2002
 * @see        Parser
 * @see      Defaults
 * @revision    $Revision$
 */
public class FixLenDataParser implements Parser {

	private BadDataFormatExceptionHandler handlerBDFE;
	private ByteBuffer dataBuffer;
	private ByteBuffer fieldBuffer;
	//private CharBuffer fieldStringBuffer;  //not used
	private DataRecordMetadata metadata;

	private ReadableByteChannel reader;
	private CharsetDecoder decoder;
	private int recordCounter;
	private int fieldLengths[];

	static Log logger = LogFactory.getLog(FixLenDataParser.class);
	
	// Attributes

	/**
	 *Constructor for the FixLenDataParser object
	 *
	 * @since    August 21, 2002
	 */
	public FixLenDataParser() {
		dataBuffer = ByteBuffer.allocateDirect(Defaults.DEFAULT_INTERNAL_IO_BUFFER_SIZE);

		fieldBuffer = ByteBuffer.allocateDirect(Defaults.DataParser.FIELD_BUFFER_LENGTH);
		decoder = Charset.forName(Defaults.DataParser.DEFAULT_CHARSET_DECODER).newDecoder();
	}


	/**
	 *Constructor for the FixLenDataParser object
	 *
	 * @param  charsetDecoder  Description of Parameter
	 * @since                  August 21, 2002
	 */
	public FixLenDataParser(String charsetDecoder) {
		dataBuffer = ByteBuffer.allocateDirect(Defaults.DEFAULT_INTERNAL_IO_BUFFER_SIZE);

		fieldBuffer = ByteBuffer.allocateDirect(Defaults.DataParser.FIELD_BUFFER_LENGTH);
		decoder = Charset.forName(charsetDecoder).newDecoder();
	}


	/**
	 *  Gets the Next attribute of the FixLenDataParser object
	 *
	 * @return                  The Next value
	 * @exception  IOException  Description of Exception
	 * @since                   August 21, 2002
	 */
	public DataRecord getNext() throws JetelException {
		// create a new data record
		DataRecord record = new DataRecord(metadata);

		record.init();

		record = parseNext(record);
		if(handlerBDFE != null ) {  //use handler only if configured
			while(handlerBDFE.isThrowException()) {
				handlerBDFE.handleException(record);
				record = parseNext(record);
			}
		}
		return record;
	}


	/**
	 *  Gets the Next attribute of the FixLenDataParser object
	 *
	 * @param  record           Description of Parameter
	 * @return                  The Next value
	 * @exception  IOException  Description of Exception
	 * @since                   August 21, 2002
	 */
	public DataRecord getNext(DataRecord record) throws JetelException {
		record = parseNext(record);
		if(handlerBDFE != null ) {  //use handler only if configured
			while(handlerBDFE.isThrowException()) {
				handlerBDFE.handleException(record);
				record = parseNext(record);
			}
		}
		return record;
	}


	/**
	 *  Description of the Method
	 *
	 * @param  in         Description of Parameter
	 * @param  _metadata  Description of Parameter
	 * @since             August 21, 2002
	 */
	public void open(Object in, DataRecordMetadata _metadata) {
		CoderResult result;
		this.metadata = _metadata;

		reader = Channels.newChannel((InputStream)in);

		// create array of field sizes & initialize them
		fieldLengths = new int[metadata.getNumFields()];
		for (int i = 0; i < metadata.getNumFields(); i++) {
			fieldLengths[i] = metadata.getField(i).getSize();
		}
		decoder.reset();
		// reset CharsetDecoder
		recordCounter = 1;
		// reset record counter
		dataBuffer.clear();
		dataBuffer.compact();
	}

{'fraction_non_alphanumeric': 0.08911756140180566, 'fraction_numerical': 0.012037666245995534, 'mean_word_length': 3.4869337979094075, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3632698', 'n_tokens_mistral': 3199, 'n_tokens_neox': 2844, 'n_words': 921}