<commit_before> /* Copyright (c) 2007 Pentaho Corporation.  All rights reserved. 
 * This software was developed by Pentaho Corporation and is provided under the terms 
 * of the GNU Lesser General Public License, Version 2.1. You may not use 
 * this file except in compliance with the license. If you need a copy of the license, 
 * please go to http://www.gnu.org/licenses/lgpl-2.1.txt. The Original Code is Pentaho 
 * Data Integration.  The Initial Developer is Pentaho Corporation.
 *
 * Software distributed under the GNU Lesser Public License is distributed on an "AS IS" 
 * basis, WITHOUT WARRANTY OF ANY KIND, either express or  implied. Please refer to 
 * the license for the specific language governing your rights and limitations.*/
 
package org.pentaho.di.trans.steps.databaselookup;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.Enumeration;
import java.util.Hashtable;
import java.util.List;

import org.pentaho.di.core.Const;
import org.pentaho.di.core.RowMetaAndData;
import org.pentaho.di.core.TimedRow;
import org.pentaho.di.core.database.Database;
import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.exception.KettleStepException;
import org.pentaho.di.core.row.RowDataUtil;
import org.pentaho.di.core.row.RowMeta;
import org.pentaho.di.core.row.RowMetaInterface;
import org.pentaho.di.core.row.ValueMeta;
import org.pentaho.di.core.row.ValueMetaInterface;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.BaseStep;
import org.pentaho.di.trans.step.StepDataInterface;
import org.pentaho.di.trans.step.StepInterface;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.step.StepMetaInterface;

/**
 * Looks up values in a database using keys from input streams.
 * 
 * @author Matt
 * @since 26-apr-2003
 */
public class DatabaseLookup extends BaseStep implements StepInterface
{
	private DatabaseLookupMeta meta;
	private DatabaseLookupData data;

	public DatabaseLookup(StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, Trans trans)
	{
		super(stepMeta, stepDataInterface, copyNr, transMeta, trans);
	}

	/**
	 * Performs the lookup based on the meta-data and the input row.
	 * @param row The row to use as lookup data and the row to add the returned lookup fields to
	 * @return the resulting row after the lookup values where added
	 * @throws KettleException In case something goes wrong.
	 */
	private synchronized Object[] lookupValues(RowMetaInterface inputRowMeta, Object[] row) throws KettleException
	{
		Object[] outputRow = RowDataUtil.resizeArray(row, data.outputRowMeta.size());
                
        Object[] lookupRow = new Object[data.lookupMeta.size()];
        int lookupIndex=0;
        
        for (int i=0;i<meta.getStreamKeyField1().length;i++)
		{
			if (data.keynrs[i]>=0)
			{
                ValueMetaInterface input = inputRowMeta.getValueMeta(data.keynrs[i]);
				ValueMetaInterface value = data.lookupMeta.getValueMeta(lookupIndex);
                lookupRow[lookupIndex] = row[data.keynrs[i]];
                
				// Try to convert type if needed
				if (input.getType()!=value.getType())
                {
                    lookupRow[lookupIndex] = value.convertData(input, lookupRow[lookupIndex]);
                }
                lookupIndex++;
			}
			if (data.keynrs2[i]>=0)
			{
                ValueMetaInterface input = inputRowMeta.getValueMeta(data.keynrs2[i]);
                ValueMetaInterface value = data.lookupMeta.getValueMeta(lookupIndex);
                lookupRow[lookupIndex] = row[data.keynrs2[i]];
                
                // Try to convert type if needed
                if (input.getType()!=value.getType())
                {
                    lookupRow[lookupIndex] = value.convertData(input, lookupRow[lookupIndex]);
                }
                lookupIndex++;
			}
		}

        Object[] add = null;
        boolean cache_now=false;
        boolean cacheHit = false;

		// First, check if we looked up before
		if (meta.isCached())
        {
			add = getRowFromCache(data.lookupMeta, lookupRow);
			if (add!=null) 
			{
				cacheHit=true;
			}
        }
		else add=null; 

		if (add==null)
		{
			if (!meta.isLoadingAllDataInCache() || data.hasDBCondition) { // do not go to the database when all rows are in (exception LIKE operator)
				if (log.isRowLevel()) logRowlevel(Messages.getString("DatabaseLookup.Log.AddedValuesToLookupRow1")+meta.getStreamKeyField1().length+Messages.getString("DatabaseLookup.Log.AddedValuesToLookupRow2")+data.lookupMeta.getString(lookupRow)); //$NON-NLS-1$ //$NON-NLS-2$

				data.db.setValuesLookup(data.lookupMeta, lookupRow);
				add = data.db.getLookup(meta.isFailingOnMultipleResults());
				cache_now=true;
			}
		}

		if (add==null) // nothing was found, unknown code: add default values
		{
			if (meta.isEatingRowOnLookupFailure())
			{
				return null;
			}
			if (getStepMeta().isDoingErrorHandling())
			{
                putError(getInputRowMeta(), row, 1L, "No lookup found", null, "DBL001");

                // return false else we would still be processed.
                return null;
			}
			
			if (log.isRowLevel()) logRowlevel(Messages.getString("DatabaseLookup.Log.NoResultsFoundAfterLookup")); //$NON-NLS-1$
            
			add=new Object[data.returnMeta.size()];
			for (int i=0;i<meta.getReturnValueField().length;i++)
			{
				if (data.nullif[i]!=null)
				{
					add[i] = data.nullif[i];
				}
				else
				{
					add[i] = null;			
				}
			}
		}
        else
        {
        	if (log.isRowLevel()) logRowlevel(Messages.getString("DatabaseLookup.Log.FoundResultsAfterLookup")+add); //$NON-NLS-1$

        	// Only verify the data types if the data comes from the DB, NOT when we have a cache hit
        	// In that case, we already know the data type is OK.
        	if (!cacheHit)
        	{
	        	int types[] = meta.getReturnValueDefaultType();
	
	        	// The assumption here is that the types are in the same order
	        	// as the returned lookup row, but since we make the lookup row
	        	// that should not be a problem.
	            //
	            for (int i=0;i<types.length;i++)
	        	{  
	        		ValueMetaInterface returned = data.db.getReturnRowMeta().getValueMeta(i);
	                ValueMetaInterface expected = data.returnMeta.getValueMeta(i);
	                
	                if ( returned != null && types[i] > 0 && types[i] !=  returned.getType() )
	        		{
	        			// Set the type to the default return type
	        		    add[i] = expected.convertData(returned, add[i]);
	        		}
	        	}
        	}
        } <commit_msg>Database Lookup did not record the number of records input.<commit_after> /* Copyright (c) 2007 Pentaho Corporation.  All rights reserved. 
 * This software was developed by Pentaho Corporation and is provided under the terms 
 * of the GNU Lesser General Public License, Version 2.1. You may not use 
 * this file except in compliance with the license. If you need a copy of the license, 
 * please go to http://www.gnu.org/licenses/lgpl-2.1.txt. The Original Code is Pentaho 
 * Data Integration.  The Initial Developer is Pentaho Corporation.
 *
 * Software distributed under the GNU Lesser Public License is distributed on an "AS IS" 
 * basis, WITHOUT WARRANTY OF ANY KIND, either express or  implied. Please refer to 
 * the license for the specific language governing your rights and limitations.*/
 
package org.pentaho.di.trans.steps.databaselookup;

import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.Enumeration;
import java.util.Hashtable;
import java.util.List;

import org.pentaho.di.core.Const;
import org.pentaho.di.core.RowMetaAndData;
import org.pentaho.di.core.TimedRow;
import org.pentaho.di.core.database.Database;
import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.exception.KettleStepException;
import org.pentaho.di.core.row.RowDataUtil;
import org.pentaho.di.core.row.RowMeta;
import org.pentaho.di.core.row.RowMetaInterface;
import org.pentaho.di.core.row.ValueMeta;
import org.pentaho.di.core.row.ValueMetaInterface;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.BaseStep;
import org.pentaho.di.trans.step.StepDataInterface;
import org.pentaho.di.trans.step.StepInterface;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.step.StepMetaInterface;

/**
 * Looks up values in a database using keys from input streams.
 * 
 * @author Matt
 * @since 26-apr-2003
 */
public class DatabaseLookup extends BaseStep implements StepInterface
{
	private DatabaseLookupMeta meta;
	private DatabaseLookupData data;

	public DatabaseLookup(StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, Trans trans)
	{
		super(stepMeta, stepDataInterface, copyNr, transMeta, trans);
	}

	/**
	 * Performs the lookup based on the meta-data and the input row.
	 * @param row The row to use as lookup data and the row to add the returned lookup fields to
	 * @return the resulting row after the lookup values where added
	 * @throws KettleException In case something goes wrong.
	 */
	private synchronized Object[] lookupValues(RowMetaInterface inputRowMeta, Object[] row) throws KettleException
	{
		Object[] outputRow = RowDataUtil.resizeArray(row, data.outputRowMeta.size());
                
        Object[] lookupRow = new Object[data.lookupMeta.size()];
        int lookupIndex=0;
        
        for (int i=0;i<meta.getStreamKeyField1().length;i++)
		{
			if (data.keynrs[i]>=0)
			{
                ValueMetaInterface input = inputRowMeta.getValueMeta(data.keynrs[i]);
				ValueMetaInterface value = data.lookupMeta.getValueMeta(lookupIndex);
                lookupRow[lookupIndex] = row[data.keynrs[i]];
                
				// Try to convert type if needed
				if (input.getType()!=value.getType())
                {
                    lookupRow[lookupIndex] = value.convertData(input, lookupRow[lookupIndex]);
                }
                lookupIndex++;
			}
			if (data.keynrs2[i]>=0)
			{
                ValueMetaInterface input = inputRowMeta.getValueMeta(data.keynrs2[i]);
                ValueMetaInterface value = data.lookupMeta.getValueMeta(lookupIndex);
                lookupRow[lookupIndex] = row[data.keynrs2[i]];
                
                // Try to convert type if needed
                if (input.getType()!=value.getType())
                {
                    lookupRow[lookupIndex] = value.convertData(input, lookupRow[lookupIndex]);
                }
                lookupIndex++;
			}
		}

        Object[] add = null;
        boolean cache_now=false;
        boolean cacheHit = false;

		// First, check if we looked up before
		if (meta.isCached())
        {
			add = getRowFromCache(data.lookupMeta, lookupRow);
			if (add!=null) 
			{
				cacheHit=true;
			}
        }
		else add=null; 

		if (add==null)
		{
			if (!meta.isLoadingAllDataInCache() || data.hasDBCondition) { // do not go to the database when all rows are in (exception LIKE operator)
				if (log.isRowLevel()) logRowlevel(Messages.getString("DatabaseLookup.Log.AddedValuesToLookupRow1")+meta.getStreamKeyField1().length+Messages.getString("DatabaseLookup.Log.AddedValuesToLookupRow2")+data.lookupMeta.getString(lookupRow)); //$NON-NLS-1$ //$NON-NLS-2$

				data.db.setValuesLookup(data.lookupMeta, lookupRow);
				add = data.db.getLookup(meta.isFailingOnMultipleResults());
				cache_now=true;
			}
		}

		if (add==null) // nothing was found, unknown code: add default values
		{
			if (meta.isEatingRowOnLookupFailure())
			{
				return null;
			}
			if (getStepMeta().isDoingErrorHandling())
			{
                putError(getInputRowMeta(), row, 1L, "No lookup found", null, "DBL001");

                // return false else we would still be processed.
                return null;
			}
			
			if (log.isRowLevel()) logRowlevel(Messages.getString("DatabaseLookup.Log.NoResultsFoundAfterLookup")); //$NON-NLS-1$
            
			add=new Object[data.returnMeta.size()];
			for (int i=0;i<meta.getReturnValueField().length;i++)
			{
				if (data.nullif[i]!=null)
				{
					add[i] = data.nullif[i];
				}
				else
				{
					add[i] = null;			
				}
			}
		}
        else
        {
        	if (log.isRowLevel()) logRowlevel(Messages.getString("DatabaseLookup.Log.FoundResultsAfterLookup")+add); //$NON-NLS-1$

        	// Only verify the data types if the data comes from the DB, NOT when we have a cache hit
        	// In that case, we already know the data type is OK.
        	if (!cacheHit)
        	{
                incrementLinesInput();

	        	int types[] = meta.getReturnValueDefaultType();
	
	        	// The assumption here is that the types are in the same order
	        	// as the returned lookup row, but since we make the lookup row
	        	// that should not be a problem.
	            //
	            for (int i=0;i<types.length;i++)
	        	{  
	        		ValueMetaInterface returned = data.db.getReturnRowMeta().getValueMeta(i);
	                ValueMetaInterface expected = data.returnMeta.getValueMeta(i);
	                
	                if ( returned != null && types[i] > 0 && types[i] !=  returned.getType() )
	        		{
	        			// Set the type to the default return type
	        		    add[i] = expected.convertData(returned, add[i]);
	        		}
	        	}
        	}
        } 
{'fraction_non_alphanumeric': 0.1065166569257744, 'fraction_numerical': 0.005260081823495032, 'mean_word_length': 2.7309893704006543, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7391672', 'n_tokens_mistral': 4378, 'n_tokens_neox': 3988, 'n_words': 1042}