<commit_before>/**
 * IBM Confidential
 * OCO Source Materials
 * (C) Copyright IBM Corp. 2010, 2015
 * The source code for this program is not published or otherwise divested of its trade secrets, irrespective of what has been deposited with the U.S. Copyright Office.
 */

package com.ibm.bi.dml.runtime.instructions;

import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;

import com.ibm.bi.dml.api.DMLScript;
import com.ibm.bi.dml.hops.OptimizerUtils;
import com.ibm.bi.dml.lops.DataGen;
import com.ibm.bi.dml.lops.Lop;
import com.ibm.bi.dml.lops.compile.JobType;
import com.ibm.bi.dml.parser.Expression.DataType;
import com.ibm.bi.dml.runtime.DMLRuntimeException;
import com.ibm.bi.dml.runtime.controlprogram.ExecutionContext;
import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
import com.ibm.bi.dml.runtime.instructions.cp.Data;
import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
import com.ibm.bi.dml.runtime.matrix.MatrixDimensionsMetaData;
import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
import com.ibm.bi.dml.runtime.matrix.data.NumItemsByEachReducerMetaData;
import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
import com.ibm.bi.dml.runtime.util.UtilFunctions;

/*
---------------------------------------------------------------------------------------
JobType       Rand    RecordReader    Mapper     Shuffle   AggInReducer  OtherInReducer
---------------------------------------------------------------------------------------
GMR                       *            *                         *              *    
RAND            *                      *                         *              *    
REBLOCK                                *            *                           *
MMCJ                                   *            *
MMRJ                                   *            *                           *
CM_COV                                 *            *
GROUPED_AGG                                         *                           *
COMBINE                                             *    
SORT                                   *            *
PARTITION
---------------------------------------------------------------------------------------
 */

public class MRJobInstruction extends Instruction
{
	@SuppressWarnings("unused")
	private static final String _COPYRIGHT = "Licensed Materials - Property of IBM\n(C) Copyright IBM Corp. 2010, 2015\n" +
                                             "US Government Users Restricted Rights - Use, duplication  disclosure restricted by GSA ADP Schedule Contract with IBM Corp.";
	
	//public enum JobType {MMCJ, MMRJ, GMR, Partition, RAND, ReBlock, SortKeys, Combine, CMCOV, GroupedAgg}; 
	private JobType jobType;
	
	private String _randInstructions = "";
	private String _recordReaderInstructions = "";
	private String _mapperInstructions = ""; 
	private String _shuffleInstructions = ""; 
	private String _aggInstructions = "";
	private String _otherInstructions = "";
	
	private String[] inputVars;
	private String[] outputVars;
	private byte [] _resultIndices;
	
	private int iv_numReducers;
	private int iv_replication;
	private String dimsUnknownFilePrefix;

	private double _mapperMem = -1;
	
	/**
	 * This structure contains the DML script line number
	 * of each MR instructions within this MR job
	 */
	private ArrayList<Integer> MRJobInstructionsLineNumbers;

	
	/*
	 * Following attributes are populated by pulling out information from Symbol Table.
	 * This is done just before a job is submitted/spawned.
	 */
	private String[] inputs;
	private InputInfo[] inputInfos;
	private long[] rlens;
	private long[] clens;
	private int[] brlens;
	private int[] bclens;
	private String[] outputs;
	private OutputInfo[] outputInfos;

	// Member variables to store partitioning-related information for all input matrices 
	private boolean[] partitioned;
	private PDataPartitionFormat[] pformats;
	private int[] psizes;
	
	/*
	 * These members store references to MatrixObjects corresponding to different 
	 * MATRIX variables in inputVars and outputVars, respectively. Note that the 
	 * references to SCALAR input variables are not stored in <code>inputMatrices</code>.
	 * Every reference in <code>outputMatrices</code> is always points to MATRIX 
	 * since MR jobs always produces matrices. 
	 */
	private MatrixObject[] inputMatrices, outputMatrices;

	// Indicates the data type of inputVars
	private DataType[] inputDataTypes;

	
	/**
	 * Constructor
	 * @param instType
	 * @param type
	 */
	
	public MRJobInstruction(JobType type)
	{
		setType(Instruction.INSTRUCTION_TYPE.MAPREDUCE_JOB);
		jobType = type;	
	}
	
	/**
	 * (deep) Copy constructor, primarily used in parfor.
	 * Additionally, replace all occurrences of <code>srcPattern</code> with <code>targetPattern</code>
	 * @throws IllegalAccessException 
	 * @throws IllegalArgumentException 
	 * 
	 */
	public MRJobInstruction(MRJobInstruction that) 
		throws IllegalArgumentException, IllegalAccessException 
	{
		this(that.getJobType());
		Class<MRJobInstruction> cla = MRJobInstruction.class;
		
		//copy fields
		Field[] fields = cla.getDeclaredFields();
		for( Field f : fields )
		{
			f.setAccessible(true);
			if(!Modifier.isStatic(f.getModifiers()))
				f.set(this, f.get(that));
		}
		
		//replace inputs/outputs (arrays)
		inputVars = that.inputVars.clone();
		outputVars = that.outputVars.clone();
		_resultIndices = that._resultIndices.clone();
	}	
	
	
	
	public JobType getJobType()
	{
		return jobType;
	}

	public String getIv_instructionsInMapper()
	{
		return _mapperInstructions;
	}
	
	public void setIv_instructionsInMapper(String inst)<commit_msg>55555: SystemML Engine Enhancements - General overhead reduction parfor initialize local par workers (shallow copy live vars, no reflection copy mr jobs)<commit_after>/**
 * IBM Confidential
 * OCO Source Materials
 * (C) Copyright IBM Corp. 2010, 2015
 * The source code for this program is not published or otherwise divested of its trade secrets, irrespective of what has been deposited with the U.S. Copyright Office.
 */

package com.ibm.bi.dml.runtime.instructions;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;

import com.ibm.bi.dml.api.DMLScript;
import com.ibm.bi.dml.hops.OptimizerUtils;
import com.ibm.bi.dml.lops.DataGen;
import com.ibm.bi.dml.lops.Lop;
import com.ibm.bi.dml.lops.compile.JobType;
import com.ibm.bi.dml.parser.Expression.DataType;
import com.ibm.bi.dml.runtime.DMLRuntimeException;
import com.ibm.bi.dml.runtime.controlprogram.ExecutionContext;
import com.ibm.bi.dml.runtime.controlprogram.ParForProgramBlock.PDataPartitionFormat;
import com.ibm.bi.dml.runtime.controlprogram.caching.MatrixObject;
import com.ibm.bi.dml.runtime.controlprogram.parfor.ProgramConverter;
import com.ibm.bi.dml.runtime.instructions.cp.Data;
import com.ibm.bi.dml.runtime.matrix.MatrixCharacteristics;
import com.ibm.bi.dml.runtime.matrix.MatrixDimensionsMetaData;
import com.ibm.bi.dml.runtime.matrix.MatrixFormatMetaData;
import com.ibm.bi.dml.runtime.matrix.data.InputInfo;
import com.ibm.bi.dml.runtime.matrix.data.NumItemsByEachReducerMetaData;
import com.ibm.bi.dml.runtime.matrix.data.OutputInfo;
import com.ibm.bi.dml.runtime.matrix.mapred.MRJobConfiguration;
import com.ibm.bi.dml.runtime.util.UtilFunctions;

/*
---------------------------------------------------------------------------------------
JobType       Rand    RecordReader    Mapper     Shuffle   AggInReducer  OtherInReducer
---------------------------------------------------------------------------------------
GMR                       *            *                         *              *    
RAND            *                      *                         *              *    
REBLOCK                                *            *                           *
MMCJ                                   *            *
MMRJ                                   *            *                           *
CM_COV                                 *            *
GROUPED_AGG                                         *                           *
COMBINE                                             *    
SORT                                   *            *
PARTITION
---------------------------------------------------------------------------------------
 */

public class MRJobInstruction extends Instruction
{
	@SuppressWarnings("unused")
	private static final String _COPYRIGHT = "Licensed Materials - Property of IBM\n(C) Copyright IBM Corp. 2010, 2015\n" +
                                             "US Government Users Restricted Rights - Use, duplication  disclosure restricted by GSA ADP Schedule Contract with IBM Corp.";
	
	//public enum JobType {MMCJ, MMRJ, GMR, Partition, RAND, ReBlock, SortKeys, Combine, CMCOV, GroupedAgg}; 
	private JobType jobType;
	
	private String _randInstructions = "";
	private String _recordReaderInstructions = "";
	private String _mapperInstructions = ""; 
	private String _shuffleInstructions = ""; 
	private String _aggInstructions = "";
	private String _otherInstructions = "";
	
	private String[] inputVars;
	private String[] outputVars;
	private byte [] _resultIndices;
	
	private int iv_numReducers;
	private int iv_replication;
	private String dimsUnknownFilePrefix;

	private double _mapperMem = -1;
	
	/**
	 * This structure contains the DML script line number
	 * of each MR instructions within this MR job
	 */
	private ArrayList<Integer> MRJobInstructionsLineNumbers;

	
	/*
	 * Following attributes are populated by pulling out information from Symbol Table.
	 * This is done just before a job is submitted/spawned.
	 */
	private String[] inputs;
	private InputInfo[] inputInfos;
	private long[] rlens;
	private long[] clens;
	private int[] brlens;
	private int[] bclens;
	private String[] outputs;
	private OutputInfo[] outputInfos;

	// Member variables to store partitioning-related information for all input matrices 
	private boolean[] partitioned;
	private PDataPartitionFormat[] pformats;
	private int[] psizes;
	
	/*
	 * These members store references to MatrixObjects corresponding to different 
	 * MATRIX variables in inputVars and outputVars, respectively. Note that the 
	 * references to SCALAR input variables are not stored in <code>inputMatrices</code>.
	 * Every reference in <code>outputMatrices</code> is always points to MATRIX 
	 * since MR jobs always produces matrices. 
	 */
	private MatrixObject[] inputMatrices, outputMatrices;

	// Indicates the data type of inputVars
	private DataType[] inputDataTypes;

	
	/**
	 * Constructor
	 * @param instType
	 * @param type
	 */
	
	public MRJobInstruction(JobType type)
	{
		setType(Instruction.INSTRUCTION_TYPE.MAPREDUCE_JOB);
		jobType = type;	
	}
	
	/**
	 * (deep) Copy constructor, primarily used in parfor.
	 * Additionally, replace all occurrences of <code>srcPattern</code> with <code>targetPattern</code>
	 * @throws IllegalAccessException 
	 * @throws IllegalArgumentException 
	 * 
	 */
	public MRJobInstruction(MRJobInstruction that) 
		throws IllegalArgumentException, IllegalAccessException 
	{
		this( that.jobType );
		
		//copy all static variables (but no need to copy variables that are
		//overwritten in extractInputMatrices/extractOutputMatrices anyway)
		
		//copy basic variables
		_randInstructions         = that._randInstructions;
		_recordReaderInstructions = that._recordReaderInstructions;
		_mapperInstructions       = that._mapperInstructions; 
		_shuffleInstructions      = that._shuffleInstructions; 
		_aggInstructions          = that._aggInstructions;
		_otherInstructions        = that._otherInstructions;
		iv_numReducers            = that.iv_numReducers;
		iv_replication            = that.iv_replication;
		dimsUnknownFilePrefix     = that.dimsUnknownFilePrefix;
		_mapperMem                = that._mapperMem;
		MRJobInstructionsLineNumbers = that.MRJobInstructionsLineNumbers;
		
		//copy array variables (via clone)
		inputVars = that.inputVars.clone();
		outputVars = that.outputVars.clone();
		_resultIndices = that._resultIndices.clone();
	}	
	
	public JobType getJobType()
	{
		return jobType;
	}

	public String getIv_instructionsInMapper()
	{
		return _mapperInstructions;
	}
	
	public void setIv_instructionsInMapper(String inst)
{'fraction_non_alphanumeric': 0.12196087748475488, 'fraction_numerical': 0.0030886196246139226, 'mean_word_length': 2.4129729729729728, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 48, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1895880', 'n_tokens_mistral': 3669, 'n_tokens_neox': 3410, 'n_words': 877}