<commit_before> /* Copyright (c) 2007 Pentaho Corporation.  All rights reserved. 
 * This software was developed by Pentaho Corporation and is provided under the terms 
 * of the GNU Lesser General Public License, Version 2.1. You may not use 
 * this file except in compliance with the license. If you need a copy of the license, 
 * please go to http://www.gnu.org/licenses/lgpl-2.1.txt. The Original Code is Pentaho 
 * Data Integration.  The Initial Developer is Pentaho Corporation.
 *
 * Software distributed under the GNU Lesser Public License is distributed on an "AS IS" 
 * basis, WITHOUT WARRANTY OF ANY KIND, either express or  implied. Please refer to 
 * the license for the specific language governing your rights and limitations.*/
 
package org.pentaho.di.trans.steps.groupby;

import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.math.BigDecimal;
import java.net.SocketTimeoutException;
import java.util.ArrayList;

import org.pentaho.di.core.Const;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.exception.KettleFileException;
import org.pentaho.di.core.exception.KettleValueException;
import org.pentaho.di.core.row.RowDataUtil;
import org.pentaho.di.core.row.RowMeta;
import org.pentaho.di.core.row.RowMetaInterface;
import org.pentaho.di.core.row.ValueDataUtil;
import org.pentaho.di.core.row.ValueMeta;
import org.pentaho.di.core.row.ValueMetaInterface;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.BaseStep;
import org.pentaho.di.trans.step.StepDataInterface;
import org.pentaho.di.trans.step.StepInterface;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.step.StepMetaInterface;


/**
 * Groups informations based on aggregation rules. (sum, count, ...)
 * 
 * @author Matt
 * @since 2-jun-2003
 */
public class GroupBy extends BaseStep implements StepInterface
{
	private GroupByMeta meta;
	private GroupByData data;
	
	public GroupBy(StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, Trans trans)
	{
		super(stepMeta, stepDataInterface, copyNr, transMeta, trans);
		
		meta=(GroupByMeta)getStepMeta().getStepMetaInterface();
		data=(GroupByData)stepDataInterface;
	}
	
	public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException
	{
        meta=(GroupByMeta)smi;
        data=(GroupByData)sdi;
        
        Object[] r=getRow();    // get row!
        
		
        if (first)
		{
        	// What is the output looking like?
        	// 
        	data.inputRowMeta = getInputRowMeta();
        	
        	// In case we have 0 input rows, we still want to send out a single row aggregate
        	// However... the problem then is that we don't know the layout from receiving it from the previous step over the row set.
        	// So we need to calculated based on the metadata...
        	//
        	if (data.inputRowMeta==null)
        	{
        		data.inputRowMeta = getTransMeta().getPrevStepFields(getStepMeta());
        	}
        	
        	data.outputRowMeta = data.inputRowMeta.clone();
        	meta.getFields(data.outputRowMeta, getStepname(), null, null, this);
        	
        	// Do all the work we can beforehand
        	// Calculate indexes, loop up fields, etc.
        	//
			data.counts     = new long[meta.getSubjectField().length];
			data.subjectnrs = new int[meta.getSubjectField().length];

			data.cumulativeSumSourceIndexes = new ArrayList<Integer>();
			data.cumulativeSumTargetIndexes = new ArrayList<Integer>();
			
			data.cumulativeAvgSourceIndexes = new ArrayList<Integer>();
			data.cumulativeAvgTargetIndexes = new ArrayList<Integer>();
			
			for (int i=0;i<meta.getSubjectField().length;i++)
			{
				data.subjectnrs[i] = data.inputRowMeta.indexOfValue(meta.getSubjectField()[i]);
				if (data.subjectnrs[i]<0)
				{
					logError(Messages.getString("GroupBy.Log.AggregateSubjectFieldCouldNotFound",meta.getSubjectField()[i])); //$NON-NLS-1$ //$NON-NLS-2$
					setErrors(1);
					stopAll();
					return false;
				}
				
				if (meta.getAggregateType()[i]==GroupByMeta.TYPE_GROUP_CUMULATIVE_SUM)
				{
					data.cumulativeSumSourceIndexes.add(data.subjectnrs[i]);
					
					// The position of the target in the output row is the input row size + i
					//
					data.cumulativeSumTargetIndexes.add(getInputRowMeta().size()+i);
				}
				if (meta.getAggregateType()[i]==GroupByMeta.TYPE_GROUP_CUMULATIVE_AVERAGE)
				{
					data.cumulativeAvgSourceIndexes.add(data.subjectnrs[i]);

					// The position of the target in the output row is the input row size + i
					//
					data.cumulativeAvgTargetIndexes.add(getInputRowMeta().size()+i);
				}
				
				data.previousSums = new Object[data.cumulativeSumTargetIndexes.size()];
				
				data.previousAvgSum = new Object[data.cumulativeAvgTargetIndexes.size()];
				data.previousAvgCount = new long[data.cumulativeAvgTargetIndexes.size()];
			}
			
			data.groupnrs = new int[meta.getGroupField().length];
			for (int i=0;i<meta.getGroupField().length;i++)
			{
				data.groupnrs[i] = data.inputRowMeta.indexOfValue(meta.getGroupField()[i]);
				if (data.groupnrs[i]<0)
				{
					logError(Messages.getString("GroupBy.Log.GroupFieldCouldNotFound",meta.getGroupField()[i])); //$NON-NLS-1$ //$NON-NLS-2$
					setErrors(1);
					stopAll();
					return false;
				}				
			}
			
			// Create a metadata value for the counter Integers
			//<commit_msg>PDI-739 : Group By step : NPE when "include all rows" option is used without any aggregations<commit_after> /* Copyright (c) 2007 Pentaho Corporation.  All rights reserved. 
 * This software was developed by Pentaho Corporation and is provided under the terms 
 * of the GNU Lesser General Public License, Version 2.1. You may not use 
 * this file except in compliance with the license. If you need a copy of the license, 
 * please go to http://www.gnu.org/licenses/lgpl-2.1.txt. The Original Code is Pentaho 
 * Data Integration.  The Initial Developer is Pentaho Corporation.
 *
 * Software distributed under the GNU Lesser Public License is distributed on an "AS IS" 
 * basis, WITHOUT WARRANTY OF ANY KIND, either express or  implied. Please refer to 
 * the license for the specific language governing your rights and limitations.*/
 
package org.pentaho.di.trans.steps.groupby;

import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.math.BigDecimal;
import java.net.SocketTimeoutException;
import java.util.ArrayList;

import org.pentaho.di.core.Const;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.exception.KettleFileException;
import org.pentaho.di.core.exception.KettleValueException;
import org.pentaho.di.core.row.RowDataUtil;
import org.pentaho.di.core.row.RowMeta;
import org.pentaho.di.core.row.RowMetaInterface;
import org.pentaho.di.core.row.ValueDataUtil;
import org.pentaho.di.core.row.ValueMeta;
import org.pentaho.di.core.row.ValueMetaInterface;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.BaseStep;
import org.pentaho.di.trans.step.StepDataInterface;
import org.pentaho.di.trans.step.StepInterface;
import org.pentaho.di.trans.step.StepMeta;
import org.pentaho.di.trans.step.StepMetaInterface;


/**
 * Groups informations based on aggregation rules. (sum, count, ...)
 * 
 * @author Matt
 * @since 2-jun-2003
 */
public class GroupBy extends BaseStep implements StepInterface
{
	private GroupByMeta meta;
	private GroupByData data;
	
	public GroupBy(StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta, Trans trans)
	{
		super(stepMeta, stepDataInterface, copyNr, transMeta, trans);
		
		meta=(GroupByMeta)getStepMeta().getStepMetaInterface();
		data=(GroupByData)stepDataInterface;
	}
	
	public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException
	{
        meta=(GroupByMeta)smi;
        data=(GroupByData)sdi;
        
        Object[] r=getRow();    // get row!
        
		
        if (first)
		{
        	// What is the output looking like?
        	// 
        	data.inputRowMeta = getInputRowMeta();
        	
        	// In case we have 0 input rows, we still want to send out a single row aggregate
        	// However... the problem then is that we don't know the layout from receiving it from the previous step over the row set.
        	// So we need to calculated based on the metadata...
        	//
        	if (data.inputRowMeta==null)
        	{
        		data.inputRowMeta = getTransMeta().getPrevStepFields(getStepMeta());
        	}
        	
        	data.outputRowMeta = data.inputRowMeta.clone();
        	meta.getFields(data.outputRowMeta, getStepname(), null, null, this);
        	
        	// Do all the work we can beforehand
        	// Calculate indexes, loop up fields, etc.
        	//
			data.counts     = new long[meta.getSubjectField().length];
			data.subjectnrs = new int[meta.getSubjectField().length];

			data.cumulativeSumSourceIndexes = new ArrayList<Integer>();
			data.cumulativeSumTargetIndexes = new ArrayList<Integer>();
			
			data.cumulativeAvgSourceIndexes = new ArrayList<Integer>();
			data.cumulativeAvgTargetIndexes = new ArrayList<Integer>();
			
			for (int i=0;i<meta.getSubjectField().length;i++)
			{
				data.subjectnrs[i] = data.inputRowMeta.indexOfValue(meta.getSubjectField()[i]);
				if (data.subjectnrs[i]<0)
				{
					logError(Messages.getString("GroupBy.Log.AggregateSubjectFieldCouldNotFound",meta.getSubjectField()[i])); //$NON-NLS-1$ //$NON-NLS-2$
					setErrors(1);
					stopAll();
					return false;
				}
				
				if (meta.getAggregateType()[i]==GroupByMeta.TYPE_GROUP_CUMULATIVE_SUM)
				{
					data.cumulativeSumSourceIndexes.add(data.subjectnrs[i]);
					
					// The position of the target in the output row is the input row size + i
					//
					data.cumulativeSumTargetIndexes.add(getInputRowMeta().size()+i);
				}
				if (meta.getAggregateType()[i]==GroupByMeta.TYPE_GROUP_CUMULATIVE_AVERAGE)
				{
					data.cumulativeAvgSourceIndexes.add(data.subjectnrs[i]);

					// The position of the target in the output row is the input row size + i
					//
					data.cumulativeAvgTargetIndexes.add(getInputRowMeta().size()+i);
				}
				
			}
			
			data.previousSums = new Object[data.cumulativeSumTargetIndexes.size()];
			
			data.previousAvgSum = new Object[data.cumulativeAvgTargetIndexes.size()];
			data.previousAvgCount = new long[data.cumulativeAvgTargetIndexes.size()];
			
			data.groupnrs = new int[meta.getGroupField().length];
			for (int i=0;i<meta.getGroupField().length;i++)
			{
				data.groupnrs[i] = data.inputRowMeta.indexOfValue(meta.getGroupField()[i]);
				if (data.groupnrs[i]<0)
				{
					logError(Messages.getString("GroupBy.Log.GroupFieldCouldNotFound",meta.getGroupField()[i])); //$NON-NLS-1$ //$NON-NLS-2$
					setErrors(1);
					stopAll();
					return false;
				}				
			}
			
			// Create a metadata value for the counter Integers
			//
{'fraction_non_alphanumeric': 0.11098296664300923, 'fraction_numerical': 0.0045244854506742375, 'mean_word_length': 3.4557312252964425, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7053929', 'n_tokens_mistral': 3908, 'n_tokens_neox': 3435, 'n_words': 820}