<commit_before> /* Copyright (c) 2007 Pentaho Corporation.  All rights reserved. 
 * This software was developed by Pentaho Corporation and is provided under the terms 
 * of the GNU Lesser General Public License, Version 2.1. You may not use 
 * this file except in compliance with the license. If you need a copy of the license, 
 * please go to http://www.gnu.org/licenses/lgpl-2.1.txt. The Original Code is Pentaho 
 * Data Integration.  The Initial Developer is Pentaho Corporation.
 *
 * Software distributed under the GNU Lesser Public License is distributed on an "AS IS" 
 * basis, WITHOUT WARRANTY OF ANY KIND, either express or  implied. Please refer to 
 * the license for the specific language governing your rights and limitations.*/
 
/* Modifications to original RegexEval step made by <NAME> */

package org.pentaho.di.trans.steps.regexeval;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.exception.KettleStepException;
import org.pentaho.di.core.row.RowDataUtil;
import org.pentaho.di.core.row.ValueMetaInterface;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.BaseStep;
import org.pentaho.di.trans.step.StepDataInterface;
import org.pentaho.di.trans.step.StepInterface;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.step.StepMetaInterface;


/**
 * Use regular expression to validate a field or capture new fields out of an existing field.
 * 
 * @author deinspanjer
 * @since 27-03-2008
 * @author Matt
 * @since 15-08-2007
 */
public class RegexEval extends BaseStep implements StepInterface
{
	private RegexEvalMeta meta;
	private RegexEvalData data;
	
	public RegexEval(StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta,
			Trans trans)
	{
		super(stepMeta, stepDataInterface, copyNr, transMeta, trans);
	}

    public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException
	{
		meta=(RegexEvalMeta)smi;
		data=(RegexEvalData)sdi;
		
		Object[] row = getRow();
		
		if (row==null)  // no more input to be expected...
		{
			setOutputDone();
			return false;
		}
		
		if (first) // we just got started
		{
			first=false;
			// get the RowMeta
			data.previousRowMeta = getInputRowMeta().clone();
			data.outputRowMeta = getInputRowMeta().clone();
			meta.getFields(data.outputRowMeta, getStepname(), null, null, this);
			
			data.nrExtraFields = meta.isAllowCaptureGroupsFlagSet() ? meta.getFieldName().length : 0;
			
			// Check if a Field (matcher) is given
			if (meta.getMatcher()!=null)
			{
				 // Cache the position of the Field
				if (data.indexOfFieldToEvaluate<0)
				{
					data.indexOfFieldToEvaluate = data.previousRowMeta.indexOfValue(meta.getMatcher());
					data.indexOfMatchResultField = data.outputRowMeta.indexOfValue(meta.getResultFieldName());
					
					if (data.nrExtraFields > 0)
					{
						data.indexOfFirstCaptureGroupField = data.outputRowMeta.indexOfValue(meta.getFieldName()[0]);
					}
					
					if (data.indexOfFieldToEvaluate<0)
		            {                    
		                // The field is unreachable !
						logError(Messages.getString("RegexEval.Log.ErrorFindingField")+ "[" + meta.getMatcher()+"]"); 
						throw new KettleStepException(Messages.getString("RegexEval.Exception.CouldnotFindField",meta.getMatcher())); 
					}		                
		                
					
					// Let's check that Result Field is given
					if (environmentSubstitute(meta.getResultFieldName()) == null )
					{
						//	Result field is missing !
						logError(Messages.getString("RegexEval.Log.ErrorResultFieldMissing")); 
						throw new KettleStepException(Messages.getString("RegexEval.Exception.ErrorResultFieldMissing"));
					}
		                
				}
			}
			else
			{
				// Matcher is missing !
				log.logError("Error",Messages.getString("RegexEval.Log.ErrorMatcherMissing"));
				throw new KettleStepException(Messages.getString("RegexEval.Exception.ErrorMatcherMissing")); 
			}
			
			// Now create objects to do string to data type conversion...
			data.conversionRowMeta = data.outputRowMeta.clone();
			for (ValueMetaInterface valueMeta : data.conversionRowMeta.getValueMetaList()) {
				valueMeta.setType(ValueMetaInterface.TYPE_STRING);
			}

		}
		
		// reserve room
		Object[] outputRow = RowDataUtil.allocateRowData(data.outputRowMeta.size());
				
		for (int i = 0; i < data.previousRowMeta.size(); i++)
		{
			outputRow[i] = row[i];
		}
		boolean sendToErrorRow=false;
		String errorMessage = null;
		try{
			// Get the Field value
			String fieldValue= getInputRowMeta().getString(row,data.indexOfFieldToEvaluate);
			
			if (fieldValue == null)
			{
				outputRow[data.indexOfMatchResultField] = false;
			}
			else
			{
				// Search engine
				Matcher m = data.pattern.matcher(fieldValue);
	
				// Start search
				boolean isMatch = m.matches();
				outputRow[data.indexOfMatchResultField] = isMatch;
	
				String[] values = new String[data.nrExtraFields];
	
				for (int i = 0; i < data.nrExtraFields; i++)
				{
					if (isMatch)
					{
						if (data.nrExtraFields != m.groupCount())
						{<commit_msg>PDI-1869 - If the source row is null, call convertDataFromString() anyway to apply IfNull (default) values.<commit_after> /* Copyright (c) 2007 Pentaho Corporation.  All rights reserved. 
 * This software was developed by Pentaho Corporation and is provided under the terms 
 * of the GNU Lesser General Public License, Version 2.1. You may not use 
 * this file except in compliance with the license. If you need a copy of the license, 
 * please go to http://www.gnu.org/licenses/lgpl-2.1.txt. The Original Code is Pentaho 
 * Data Integration.  The Initial Developer is Pentaho Corporation.
 *
 * Software distributed under the GNU Lesser Public License is distributed on an "AS IS" 
 * basis, WITHOUT WARRANTY OF ANY KIND, either express or  implied. Please refer to 
 * the license for the specific language governing your rights and limitations.*/
 
/* Modifications to original RegexEval step made by <NAME> */

package org.pentaho.di.trans.steps.regexeval;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.exception.KettleStepException;
import org.pentaho.di.core.row.RowDataUtil;
import org.pentaho.di.core.row.ValueMetaInterface;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.BaseStep;
import org.pentaho.di.trans.step.StepDataInterface;
import org.pentaho.di.trans.step.StepInterface;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.step.StepMetaInterface;


/**
 * Use regular expression to validate a field or capture new fields out of an existing field.
 * 
 * @author deinspanjer
 * @since 27-03-2008
 * @author Matt
 * @since 15-08-2007
 */
public class RegexEval extends BaseStep implements StepInterface
{
	private RegexEvalMeta meta;
	private RegexEvalData data;
	
	public RegexEval(StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta,
			Trans trans)
	{
		super(stepMeta, stepDataInterface, copyNr, transMeta, trans);
	}

    public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException
	{
		meta=(RegexEvalMeta)smi;
		data=(RegexEvalData)sdi;
		
		Object[] row = getRow();
		
		if (row==null)  // no more input to be expected...
		{
			setOutputDone();
			return false;
		}
		
		if (first) // we just got started
		{
			first=false;
			// get the RowMeta
			data.previousRowMeta = getInputRowMeta().clone();
			data.outputRowMeta = getInputRowMeta().clone();
			meta.getFields(data.outputRowMeta, getStepname(), null, null, this);
			
			data.nrExtraFields = meta.isAllowCaptureGroupsFlagSet() ? meta.getFieldName().length : 0;
			
			// Check if a Field (matcher) is given
			if (meta.getMatcher()!=null)
			{
				 // Cache the position of the Field
				if (data.indexOfFieldToEvaluate<0)
				{
					data.indexOfFieldToEvaluate = data.previousRowMeta.indexOfValue(meta.getMatcher());
					data.indexOfMatchResultField = data.outputRowMeta.indexOfValue(meta.getResultFieldName());
					
					if (data.nrExtraFields > 0)
					{
						data.indexOfFirstCaptureGroupField = data.outputRowMeta.indexOfValue(meta.getFieldName()[0]);
					}
					
					if (data.indexOfFieldToEvaluate<0)
		            {                    
		                // The field is unreachable !
						logError(Messages.getString("RegexEval.Log.ErrorFindingField")+ "[" + meta.getMatcher()+"]"); 
						throw new KettleStepException(Messages.getString("RegexEval.Exception.CouldnotFindField",meta.getMatcher())); 
					}		                
		                
					
					// Let's check that Result Field is given
					if (environmentSubstitute(meta.getResultFieldName()) == null )
					{
						//	Result field is missing !
						logError(Messages.getString("RegexEval.Log.ErrorResultFieldMissing")); 
						throw new KettleStepException(Messages.getString("RegexEval.Exception.ErrorResultFieldMissing"));
					}
		                
				}
			}
			else
			{
				// Matcher is missing !
				log.logError("Error",Messages.getString("RegexEval.Log.ErrorMatcherMissing"));
				throw new KettleStepException(Messages.getString("RegexEval.Exception.ErrorMatcherMissing")); 
			}
			
			// Now create objects to do string to data type conversion...
			data.conversionRowMeta = data.outputRowMeta.clone();
			for (ValueMetaInterface valueMeta : data.conversionRowMeta.getValueMetaList()) {
				valueMeta.setType(ValueMetaInterface.TYPE_STRING);
			}

		}
		
		// reserve room
		Object[] outputRow = RowDataUtil.allocateRowData(data.outputRowMeta.size());
				
		for (int i = 0; i < data.previousRowMeta.size(); i++)
		{
			outputRow[i] = row[i];
		}
		boolean sendToErrorRow=false;
		String errorMessage = null;
		try{
			// Get the Field value
			String fieldValue= getInputRowMeta().getString(row,data.indexOfFieldToEvaluate);
			
			if (fieldValue == null)
			{
				outputRow[data.indexOfMatchResultField] = false;
                for (int i = 0; i < data.nrExtraFields; i++)
                {
                    ValueMetaInterface valueMeta = data.outputRowMeta.getValueMeta(data.indexOfFirstCaptureGroupField+i); 
                    ValueMetaInterface conversionValueMeta = data.conversionRowMeta.getValueMeta(data.indexOfFirstCaptureGroupField+i);
                    Object convertedValue = valueMeta.convertDataFromString
                    (
                            null,
                            conversionValueMeta,
                            meta.getFieldNullIf()[i],
                            meta.getFieldIfNull()[i],
                            meta.getFieldTrimType()[i]
                    );
    
                    outputRow[data.indexOfFirstCaptureGroupField+i] = convertedValue;
                }
			}
			else
			{
				// Search engine
				Matcher m = data.pattern.matcher(fieldValue);
	
				// Start search
				boolean isMatch = m.matches();
				outputRow[data.indexOfMatchResultField] = isMatch;
	
				String[] values = new String[data.nrExtraFields];
	
				for (int i = 0; i < data.nrExtraFields; i++)
				{
					if (isMatch)
					{
						if (data.nrExtraFields != m.groupCount())
						{
{'fraction_non_alphanumeric': 0.10289473684210526, 'fraction_numerical': 0.005877192982456141, 'mean_word_length': 3.1099495313626533, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7388114', 'n_tokens_mistral': 3883, 'n_tokens_neox': 3211, 'n_words': 803}