<commit_before> /* Copyright (c) 2007 Pentaho Corporation.  All rights reserved. 
 * This software was developed by Pentaho Corporation and is provided under the terms 
 * of the GNU Lesser General Public License, Version 2.1. You may not use 
 * this file except in compliance with the license. If you need a copy of the license, 
 * please go to http://www.gnu.org/licenses/lgpl-2.1.txt. The Original Code is Pentaho 
 * Data Integration.  The Initial Developer is Pentaho Corporation.
 *
 * Software distributed under the GNU Lesser Public License is distributed on an "AS IS" 
 * basis, WITHOUT WARRANTY OF ANY KIND, either express or  implied. Please refer to 
 * the license for the specific language governing your rights and limitations.*/
 

package org.pentaho.di.trans.steps.accessinput;

import java.io.File;
import java.util.Date;
import java.util.List;

import org.apache.commons.vfs.FileObject;
import org.pentaho.di.core.Const;
import org.pentaho.di.core.ResultFile;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.fileinput.FileInputList;
import org.pentaho.di.core.row.RowDataUtil;
import org.pentaho.di.core.row.RowMeta;
import org.pentaho.di.core.row.RowMetaInterface;
import org.pentaho.di.core.row.ValueMetaInterface;
import org.pentaho.di.core.vfs.KettleVFS;
import org.pentaho.di.i18n.BaseMessages;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.BaseStep;
import org.pentaho.di.trans.step.StepDataInterface;
import org.pentaho.di.trans.step.StepInterface;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.step.StepMetaInterface;

import com.healthmarketscience.jackcess.Database;

/**
 * Read all Access files, convert them to rows and writes these to one or more output streams.
 * 
 * @author Samatar
 * @since 24-05-2007
 */
public class AccessInput extends BaseStep implements StepInterface
{
	private static Class<?> PKG = AccessInput.class; // for i18n purposes, needed by Translator2!!   $NON-NLS-1$

	private AccessInputMeta meta;
	private AccessInputData data;
	
	public AccessInput(StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, Trans trans)
	{
		super(stepMeta, stepDataInterface, copyNr, transMeta, trans);
	}
	
	public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException
	{
	
		Object[] r=null;
		
		boolean sendToErrorRow=false;
		String errorMessage = null;
		 
		try{
			 // Grab one row
			 Object[] outputRowData=getOneRow();
			 if (outputRowData==null)
		     {
		        setOutputDone();  // signal end to receiver(s)
		        return false; // end of data or error.
		     }
	 
			 putRow(data.outputRowMeta, outputRowData);  // copy row to output rowset(s);

			  if (meta.getRowLimit()>0 && data.rownr>meta.getRowLimit())  // limit has been reached: stop now.
		      {
		            setOutputDone();
		            return false;
		      }	
		}catch(KettleException e)
		{
			if (getStepMeta().isDoingErrorHandling())
	        {
                sendToErrorRow = true;
                errorMessage = e.toString();
	        }
			else
			{
				logError(BaseMessages.getString(PKG, "AccessInput.ErrorInStepRunning",e.getMessage())); //$NON-NLS-1$
				setErrors(1);
				stopAll();
				setOutputDone();  // signal end to receiver(s)
				return false;
			}
			if (sendToErrorRow)
	         {
				 // Simply add this row to the error row
				putError(getInputRowMeta(), r, 1, errorMessage, null, "AccessInput001");
	         }
		}
		 return true;
	}		
	private Object[] getOneRow() throws KettleException
	{
		try{
			if(meta.isFileField())
			{
				 while ((data.readrow==null || ((data.rw = data.t.getNextRow())==null)))
				 { 
					if (!openNextFile()) return null;
				 }	
			}
			else
			{
				while ((data.file==null || ((data.rw = data.t.getNextRow())==null)))
				{
			        if (!openNextFile()) return null;
				}
			}
		} catch (Exception IO)
		{
			return null;
		}

		 // Build an empty row based on the meta-data		  
		 Object[] r=buildEmptyRow();
		 
		 // Create new row	or clone
		 if(meta.isFileField())
			 System.arraycopy(data.readrow, 0, r, 0, data.readrow.length);

		 
		 try{	
			 
				// Execute for each Input field...
				for (int i=0;i<meta.getInputFields().length;i++)
				{
					// get field
					AccessInputField field= meta.getInputFields()[i];

					// Get field value
					Object obj = data.rw.get(environmentSubstitute(field.getColumn()));	
					String value;
					if (obj==null) {
						value=null;
					} else {
						value=String.valueOf(obj);
					}
					
					if(obj instanceof Date && field.getType()==ValueMetaInterface.TYPE_DATE)	
					{
						r[data.totalpreviousfields+i]=obj;
					}
					else 
					{
						// DO Trimming!
						switch (field.getTrimType())
						{
						case AccessInputField.TYPE_TRIM_LEFT:
							value = Const.ltrim(value);
							break;
						case AccessInputField.TYPE_TRIM_RIGHT:
							value = Const.rtrim(value);
							break;
						case AccessInputField.TYPE_TRIM_BOTH:
							value = Const.trim(value);
							break;
						default:
							break;
						}
					
						// DO CONVERSIONS...
						//
						ValueMetaInterface targetValueMeta = data.outputRowMeta.getValueMeta(data.totalpreviousfields+i);
						ValueMetaInterface sourceValueMeta = data.convertRowMeta.getValueMeta(data.totalpreviousfields+i);
						r[data.totalpreviousfields+i] = targetValueMeta.convertData(sourceValueMeta, value);
					}
						
					// Do we need to repeat this field if it is null?
					if (field.isRepeated())
					{
						if (data.previousRow!=null && Const.isEmpty(value))
						{
							r[data.totalpreviousfields+i] = data.previousRow[data.totalpreviousfields+i];
						}
					}	<commit_msg>PDI-2719:Access Input step destroys/mangles contents of binary columns (e.g. "OLE Object" type columns) Path provided by <NAME><commit_after> /* Copyright (c) 2007 Pentaho Corporation.  All rights reserved. 
 * This software was developed by Pentaho Corporation and is provided under the terms 
 * of the GNU Lesser General Public License, Version 2.1. You may not use 
 * this file except in compliance with the license. If you need a copy of the license, 
 * please go to http://www.gnu.org/licenses/lgpl-2.1.txt. The Original Code is Pentaho 
 * Data Integration.  The Initial Developer is Pentaho Corporation.
 *
 * Software distributed under the GNU Lesser Public License is distributed on an "AS IS" 
 * basis, WITHOUT WARRANTY OF ANY KIND, either express or  implied. Please refer to 
 * the license for the specific language governing your rights and limitations.*/
 

package org.pentaho.di.trans.steps.accessinput;

import java.io.File;
import java.util.Date;
import java.util.List;

import org.apache.commons.vfs.FileObject;
import org.pentaho.di.core.Const;
import org.pentaho.di.core.ResultFile;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.fileinput.FileInputList;
import org.pentaho.di.core.row.RowDataUtil;
import org.pentaho.di.core.row.RowMeta;
import org.pentaho.di.core.row.RowMetaInterface;
import org.pentaho.di.core.row.ValueMetaInterface;
import org.pentaho.di.core.vfs.KettleVFS;
import org.pentaho.di.i18n.BaseMessages;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.BaseStep;
import org.pentaho.di.trans.step.StepDataInterface;
import org.pentaho.di.trans.step.StepInterface;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.step.StepMetaInterface;

import com.healthmarketscience.jackcess.Database;

/**
 * Read all Access files, convert them to rows and writes these to one or more output streams.
 * 
 * @author Samatar
 * @since 24-05-2007
 */
public class AccessInput extends BaseStep implements StepInterface
{
	private static Class<?> PKG = AccessInput.class; // for i18n purposes, needed by Translator2!!   $NON-NLS-1$

	private AccessInputMeta meta;
	private AccessInputData data;
	
	public AccessInput(StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, Trans trans)
	{
		super(stepMeta, stepDataInterface, copyNr, transMeta, trans);
	}
	
	public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException
	{
	
		Object[] r=null;
		
		boolean sendToErrorRow=false;
		String errorMessage = null;
		 
		try{
			 // Grab one row
			 Object[] outputRowData=getOneRow();
			 if (outputRowData==null)
		     {
		        setOutputDone();  // signal end to receiver(s)
		        return false; // end of data or error.
		     }
	 
			 putRow(data.outputRowMeta, outputRowData);  // copy row to output rowset(s);

			  if (meta.getRowLimit()>0 && data.rownr>meta.getRowLimit())  // limit has been reached: stop now.
		      {
		            setOutputDone();
		            return false;
		      }	
		}catch(KettleException e)
		{
			if (getStepMeta().isDoingErrorHandling())
	        {
                sendToErrorRow = true;
                errorMessage = e.toString();
	        }
			else
			{
				logError(BaseMessages.getString(PKG, "AccessInput.ErrorInStepRunning",e.getMessage())); //$NON-NLS-1$
				setErrors(1);
				stopAll();
				setOutputDone();  // signal end to receiver(s)
				return false;
			}
			if (sendToErrorRow)
	         {
				 // Simply add this row to the error row
				putError(getInputRowMeta(), r, 1, errorMessage, null, "AccessInput001");
	         }
		}
		 return true;
	}		
	private Object[] getOneRow() throws KettleException
	{
		try{
			if(meta.isFileField())
			{
				 while ((data.readrow==null || ((data.rw = data.t.getNextRow())==null)))
				 { 
					if (!openNextFile()) return null;
				 }	
			}
			else
			{
				while ((data.file==null || ((data.rw = data.t.getNextRow())==null)))
				{
			        if (!openNextFile()) return null;
				}
			}
		} catch (Exception IO)
		{
			return null;
		}

		 // Build an empty row based on the meta-data		  
		 Object[] r=buildEmptyRow();
		 
		 // Create new row	or clone
		 if(meta.isFileField())
			 System.arraycopy(data.readrow, 0, r, 0, data.readrow.length);

		 
		 try{	
			 
				// Execute for each Input field...
				for (int i=0;i<meta.getInputFields().length;i++)
				{
					// get field
					AccessInputField field= meta.getInputFields()[i];

					// Get field value
					Object obj = data.rw.get(environmentSubstitute(field.getColumn()));	
						
					if(obj instanceof Date && field.getType()==ValueMetaInterface.TYPE_DATE
						|| obj instanceof byte[] && field.getType()==ValueMetaInterface.TYPE_BINARY)
					{
						r[data.totalpreviousfields+i]=obj;
					}
					else 
					{
						String value=null;
						if (obj!=null) {
							value=String.valueOf(obj);
						}
						
						// DO Trimming!
						switch (field.getTrimType())
						{
						case AccessInputField.TYPE_TRIM_LEFT:
							value = Const.ltrim(value);
							break;
						case AccessInputField.TYPE_TRIM_RIGHT:
							value = Const.rtrim(value);
							break;
						case AccessInputField.TYPE_TRIM_BOTH:
							value = Const.trim(value);
							break;
						default:
							break;
						}
					
						// DO CONVERSIONS...
						//
						ValueMetaInterface targetValueMeta = data.outputRowMeta.getValueMeta(data.totalpreviousfields+i);
						ValueMetaInterface sourceValueMeta = data.convertRowMeta.getValueMeta(data.totalpreviousfields+i);
						r[data.totalpreviousfields+i] = targetValueMeta.convertData(sourceValueMeta, value);
					}
						
					// Do we need to repeat this field if it is null?
					if (field.isRepeated())
					{
						if (data.previousRow!=null && obj==null)
						{
							r[data.totalpreviousfields+i] = data.previousRow[data.totalpreviousfields+i];
						}
					}	
{'fraction_non_alphanumeric': 0.11162196679438059, 'fraction_numerical': 0.0057896977437207325, 'mean_word_length': 2.9442578912021493, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5105445', 'n_tokens_mistral': 4451, 'n_tokens_neox': 3557, 'n_words': 898}