<commit_before> /**********************************************************************
 **                                                                   **
 **               This code belongs to the KETTLE project.            **
 **                                                                   **
 ** Kettle, from version 2.2 on, is released into the public domain   **
 ** under the Lesser GNU Public License (LGPL).                       **
 **                                                                   **
 ** For more details, please read the document LICENSE.txt, included  **
 ** in this project                                                   **
 **                                                                   **
 ** http://www.kettle.be                                              **
 ** <EMAIL>                                                    **
 **                                                                   **
 **********************************************************************/
 
package org.pentaho.di.trans.steps.xbaseinput;

import org.pentaho.di.core.Const;
import org.pentaho.di.core.ResultFile;
import org.pentaho.di.core.RowSet;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.row.RowDataUtil;
import org.pentaho.di.core.row.RowMeta;
import org.pentaho.di.core.row.RowMetaInterface;
import org.pentaho.di.core.vfs.KettleVFS;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.BaseStep;
import org.pentaho.di.trans.step.StepDataInterface;
import org.pentaho.di.trans.step.StepInterface;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.step.StepMetaInterface;

/**
 * Reads data from an XBase (dBase, foxpro, ...) file.
 * 
 * @author Matt
 * @since 8-sep-2004
 */
public class XBaseInput extends BaseStep implements StepInterface
{
	private XBaseInputMeta meta;
	private XBaseInputData data;
		
	public XBaseInput(StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, Trans trans)
	{
		super(stepMeta, stepDataInterface, copyNr, transMeta, trans);
	}
	
	public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException
	{
		meta=(XBaseInputMeta)smi;
		data=(XBaseInputData)sdi;
        
        // See if we need to get a list of files from input...
        if (first) // we just got started
        {
            first = false;
            
            // The output row meta data, what does it look like?
            //
            data.outputRowMeta = new RowMeta();
            meta.getFields(data.outputRowMeta, getStepname(), null, null, this);
            
            if (meta.isAcceptingFilenames())
            {
                // Read the files from the specified input stream...
                data.files.getFiles().clear();
                
                int idx = -1;
                
                RowSet rowSet = findInputRowSet(meta.getAcceptingStepName());
                Object[] fileRowData = getRowFrom(rowSet);
                while (fileRowData!=null)
                {
                	RowMetaInterface fileRowMeta = rowSet.getRowMeta();
                    if (idx<0)
                    {
                        idx = fileRowMeta.indexOfValue(meta.getAcceptingField());
                        if (idx<0)
                        {
                            logError(Messages.getString("XBaseInput.Log.Error.UnableToFindFilenameField", meta.getAcceptingField()));
                            setErrors(1);
                            stopAll();
                            return false;
                        }
                    }
                    try
                    {
                    	String filename = fileRowMeta.getString(fileRowData, idx);
                        data.files.addFile( KettleVFS.getFileObject(filename) );
                    }
                    catch(Exception e)
                    {
                        throw new KettleException(e);
                    }
                    
                    // Grab another row
                    //
                    fileRowData = getRowFrom(rowSet);
                }
                
                if (data.files.nrOfFiles()==0)
                {
                    logBasic(Messages.getString("XBaseInput.Log.Error.NoFilesSpecified"));
                    return false;
                }
            }

            // Open the first file & read the required rows in the buffer, stop
            // if it fails, exception will stop processLoop
            //
            openNextFile();
        }
        
        // Allocate the output row in advance, because we possibly want to add a few extra fields...
        //
        Object[] row = data.xbi.getRow( RowDataUtil.allocateRowData(data.outputRowMeta.size()) );
        while (row==null && data.fileNr < data.files.nrOfFiles()) // No more rows left in this file
        {
            openNextFile();
            row = data.xbi.getRow( RowDataUtil.allocateRowData(data.outputRowMeta.size()) );
        }
        
        if (row==null) 
        {           
            setOutputDone();  // signal end to receiver(s)
            return false; // end of data or error.
        }
        
        // OK, so we have read a line: increment the input counter
		linesInput++;
		int outputIndex = data.fields.size();

        // Possibly add a filename...
        if (meta.includeFilename())
        {
            row[outputIndex++] = data.file_dbf.getName().getURI();
        }

        // Possibly add a row number...
        if (meta.isRowNrAdded())
        {
            row[outputIndex++] = new Long(linesInput);
        }

		putRow(data.outputRowMeta, row);        // fill the rowset(s). (wait for empty)

        if (checkFeedback(linesInput)) logBasic(Messages.getString("XBaseInput.Log.LineNr")+linesInput); //$NON-NLS-1$

		return true;
	}
	
	public boolean init(StepMetaInterface smi, StepDataInterface sdi)
	{
		meta=(XBaseInputMeta)smi;
		data=(XBaseInputData)sdi;

	    if (super.init(smi, sdi))
	    {
            data.files  = meta.getTextFileList(this);
            data.fileNr = 0;
            
            if (data.files.nrOfFiles()==0 && !meta.isAcceptingFilenames())
            {
                logError(Messages.getString("XBaseInput.Log.Error.NoFilesSpecified"));
                return false;
            }
            return true;
	    }<commit_msg>Fix for PDI-156 : activate limit field.<commit_after> /**********************************************************************
 **                                                                   **
 **               This code belongs to the KETTLE project.            **
 **                                                                   **
 ** Kettle, from version 2.2 on, is released into the public domain   **
 ** under the Lesser GNU Public License (LGPL).                       **
 **                                                                   **
 ** For more details, please read the document LICENSE.txt, included  **
 ** in this project                                                   **
 **                                                                   **
 ** http://www.kettle.be                                              **
 ** <EMAIL>                                                    **
 **                                                                   **
 **********************************************************************/
 
package org.pentaho.di.trans.steps.xbaseinput;

import org.pentaho.di.core.Const;
import org.pentaho.di.core.ResultFile;
import org.pentaho.di.core.RowSet;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.row.RowDataUtil;
import org.pentaho.di.core.row.RowMeta;
import org.pentaho.di.core.row.RowMetaInterface;
import org.pentaho.di.core.vfs.KettleVFS;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.BaseStep;
import org.pentaho.di.trans.step.StepDataInterface;
import org.pentaho.di.trans.step.StepInterface;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.step.StepMetaInterface;

/**
 * Reads data from an XBase (dBase, foxpro, ...) file.
 * 
 * @author Matt
 * @since 8-sep-2004
 */
public class XBaseInput extends BaseStep implements StepInterface
{
	private XBaseInputMeta meta;
	private XBaseInputData data;
		
	public XBaseInput(StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, Trans trans)
	{
		super(stepMeta, stepDataInterface, copyNr, transMeta, trans);
	}
	
	public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException
	{
		meta=(XBaseInputMeta)smi;
		data=(XBaseInputData)sdi;
        
        // See if we need to get a list of files from input...
        if (first) // we just got started
        {
            first = false;
            
            // The output row meta data, what does it look like?
            //
            data.outputRowMeta = new RowMeta();
            meta.getFields(data.outputRowMeta, getStepname(), null, null, this);
            
            if (meta.isAcceptingFilenames())
            {
                // Read the files from the specified input stream...
                data.files.getFiles().clear();
                
                int idx = -1;
                
                RowSet rowSet = findInputRowSet(meta.getAcceptingStepName());
                Object[] fileRowData = getRowFrom(rowSet);
                while (fileRowData!=null)
                {
                	RowMetaInterface fileRowMeta = rowSet.getRowMeta();
                    if (idx<0)
                    {
                        idx = fileRowMeta.indexOfValue(meta.getAcceptingField());
                        if (idx<0)
                        {
                            logError(Messages.getString("XBaseInput.Log.Error.UnableToFindFilenameField", meta.getAcceptingField()));
                            setErrors(1);
                            stopAll();
                            return false;
                        }
                    }
                    try
                    {
                    	String filename = fileRowMeta.getString(fileRowData, idx);
                        data.files.addFile( KettleVFS.getFileObject(filename) );
                    }
                    catch(Exception e)
                    {
                        throw new KettleException(e);
                    }
                    
                    // Grab another row
                    //
                    fileRowData = getRowFrom(rowSet);
                }
                
                if (data.files.nrOfFiles()==0)
                {
                    logBasic(Messages.getString("XBaseInput.Log.Error.NoFilesSpecified"));
                    return false;
                }
            }

            // Open the first file & read the required rows in the buffer, stop
            // if it fails, exception will stop processLoop
            //
            openNextFile();
        }
        
        // Allocate the output row in advance, because we possibly want to add a few extra fields...
        //
        Object[] row = data.xbi.getRow( RowDataUtil.allocateRowData(data.outputRowMeta.size()) );
        while (row==null && data.fileNr < data.files.nrOfFiles()) // No more rows left in this file
        {
            openNextFile();
            row = data.xbi.getRow( RowDataUtil.allocateRowData(data.outputRowMeta.size()) );
        }
        
        if (row==null) 
        {           
            setOutputDone();  // signal end to receiver(s)
            return false; // end of data or error.
        }
        
        // OK, so we have read a line: increment the input counter
		linesInput++;
		int outputIndex = data.fields.size();

        // Possibly add a filename...
        if (meta.includeFilename())
        {
            row[outputIndex++] = data.file_dbf.getName().getURI();
        }

        // Possibly add a row number...
        if (meta.isRowNrAdded())
        {
            row[outputIndex++] = new Long(linesInput);
        }

		putRow(data.outputRowMeta, row);        // fill the rowset(s). (wait for empty)

        if (checkFeedback(linesInput)) logBasic(Messages.getString("XBaseInput.Log.LineNr")+linesInput); //$NON-NLS-1$

        if (meta.getRowLimit()>0 && linesInput>=meta.getRowLimit())  // limit has been reached: stop now.
        {
            setOutputDone();
            return false;
        }
        
		return true;
	}
	
	public boolean init(StepMetaInterface smi, StepDataInterface sdi)
	{
		meta=(XBaseInputMeta)smi;
		data=(XBaseInputData)sdi;

	    if (super.init(smi, sdi))
	    {
            data.files  = meta.getTextFileList(this);
            data.fileNr = 0;
            
            if (data.files.nrOfFiles()==0 && !meta.isAcceptingFilenames())
            {
                logError(Messages.getString("XBaseInput.Log.Error.NoFilesSpecified"));
                return false;
            }
            return true;
	    }
{'fraction_non_alphanumeric': 0.11754891551645685, 'fraction_numerical': 0.002578492340360989, 'mean_word_length': 1.395023610606611, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 42, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4438701', 'n_tokens_mistral': 3401, 'n_tokens_neox': 3225, 'n_words': 735}