<commit_before>package com.splicemachine.mrio.api;

import java.io.IOException;
import java.sql.SQLException;
import java.util.List;

import org.apache.derby.iapi.error.StandardException;
import org.apache.derby.iapi.sql.execute.ExecRow;
import org.apache.derby.iapi.types.RowLocation;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableInputFormat;
import org.apache.hadoop.hbase.mapreduce.TableSplit;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.util.StringUtils;
import org.apache.log4j.Logger;

import com.splicemachine.derby.impl.sql.execute.operations.scanner.TableScannerBuilder;
import com.splicemachine.mrio.MRConstants;
import com.splicemachine.utils.SpliceLogUtils;

/**
 * 
 * Input Format that requires the following items passed to it.
 * 
 * 
 *
 */
public class SMInputFormat extends InputFormat<RowLocation, ExecRow> implements Configurable {
    protected static final Logger LOG = Logger.getLogger(SMInputFormat.class);
	protected Configuration conf;
	protected HTable table;
	protected Scan scan;
	protected SMSQLUtil util;

	@Override
	public void setConf(Configuration conf) {
		    String tableName = conf.get(MRConstants.SPLICE_INPUT_TABLE_NAME);
		    String conglomerate = conf.get(MRConstants.SPLICE_INPUT_CONGLOMERATE);
			String tableScannerAsString = conf.get(MRConstants.SPLICE_SCAN_INFO);
			String jdbcString = conf.get(MRConstants.SPLICE_JDBC_STR);
			if (tableName == null && conglomerate == null) {
			    LOG.error("Table Name Supplied is null");
		    	throw new RuntimeException("Table Name Supplied is Null");
		    }
		    if (conglomerate == null) {
				if (util==null)
					util = SMSQLUtil.getInstance(jdbcString);
				if (jdbcString == null) {
					LOG.error("JDBC String Not Supplied");
					throw new RuntimeException("JDBC String Not Supplied");
				}
				try {
				conglomerate = util.getConglomID(tableName);
				conf.set(MRConstants.SPLICE_INPUT_CONGLOMERATE, conglomerate);
				} catch (SQLException e) {
					LOG.error(StringUtils.stringifyException(e));
					throw new RuntimeException(e);
				}		    	
		    }
		    try {
		      setHTable(new HTable(new Configuration(conf), conglomerate));
		    } catch (Exception e) {
		      LOG.error(StringUtils.stringifyException(e));
		    }
			if (tableScannerAsString == null) {
				if (jdbcString == null) {
					LOG.error("JDBC String Not Supplied");
					throw new RuntimeException("JDBC String Not Supplied");
				}				
				try {
					conf.set(MRConstants.SPLICE_SCAN_INFO, util.getTableScannerBuilder(tableName, null).getTableScannerBuilderBase64String());
				} catch (Exception e) {
					LOG.error(StringUtils.stringifyException(e));
					throw new RuntimeException(e);
				}
			}		    
		    this.conf = conf;
		  }

	@Override
	public Configuration getConf() {
		return conf;
	}

	@Override
	public List<InputSplit> getSplits(JobContext context) throws IOException,
			InterruptedException {
		if (LOG.isDebugEnabled())
			SpliceLogUtils.debug(LOG, "getSplits");
		TableInputFormat tableInputFormat = new TableInputFormat();
		conf.set(TableInputFormat.INPUT_TABLE,conf.get(MRConstants.SPLICE_INPUT_CONGLOMERATE));		
		tableInputFormat.setConf(conf);
		try {
			tableInputFormat.setScan(TableScannerBuilder.getTableScannerBuilderFromBase64String(conf.get(MRConstants.SPLICE_SCAN_INFO)).getScan());
		} catch (StandardException e) {			
			throw new IOException(e);
		}
		return tableInputFormat.getSplits(context);<commit_msg>Adding Split Logging<commit_after>package com.splicemachine.mrio.api;

import java.io.IOException;
import java.sql.SQLException;
import java.util.List;

import org.apache.derby.iapi.error.StandardException;
import org.apache.derby.iapi.sql.execute.ExecRow;
import org.apache.derby.iapi.types.RowLocation;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.mapreduce.TableInputFormat;
import org.apache.hadoop.hbase.mapreduce.TableSplit;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.apache.hadoop.util.StringUtils;
import org.apache.log4j.Logger;

import com.splicemachine.derby.impl.sql.execute.operations.scanner.TableScannerBuilder;
import com.splicemachine.mrio.MRConstants;
import com.splicemachine.utils.SpliceLogUtils;

/**
 * 
 * Input Format that requires the following items passed to it.
 * 
 * 
 *
 */
public class SMInputFormat extends InputFormat<RowLocation, ExecRow> implements Configurable {
    protected static final Logger LOG = Logger.getLogger(SMInputFormat.class);
	protected Configuration conf;
	protected HTable table;
	protected Scan scan;
	protected SMSQLUtil util;

	@Override
	public void setConf(Configuration conf) {
		    String tableName = conf.get(MRConstants.SPLICE_INPUT_TABLE_NAME);
		    String conglomerate = conf.get(MRConstants.SPLICE_INPUT_CONGLOMERATE);
			String tableScannerAsString = conf.get(MRConstants.SPLICE_SCAN_INFO);
			String jdbcString = conf.get(MRConstants.SPLICE_JDBC_STR);
			if (tableName == null && conglomerate == null) {
			    LOG.error("Table Name Supplied is null");
		    	throw new RuntimeException("Table Name Supplied is Null");
		    }
		    if (conglomerate == null) {
				if (util==null)
					util = SMSQLUtil.getInstance(jdbcString);
				if (jdbcString == null) {
					LOG.error("JDBC String Not Supplied");
					throw new RuntimeException("JDBC String Not Supplied");
				}
				try {
				conglomerate = util.getConglomID(tableName);
				conf.set(MRConstants.SPLICE_INPUT_CONGLOMERATE, conglomerate);
				} catch (SQLException e) {
					LOG.error(StringUtils.stringifyException(e));
					throw new RuntimeException(e);
				}		    	
		    }
		    try {
		      setHTable(new HTable(new Configuration(conf), conglomerate));
		    } catch (Exception e) {
		      LOG.error(StringUtils.stringifyException(e));
		    }
			if (tableScannerAsString == null) {
				if (jdbcString == null) {
					LOG.error("JDBC String Not Supplied");
					throw new RuntimeException("JDBC String Not Supplied");
				}				
				try {
					conf.set(MRConstants.SPLICE_SCAN_INFO, util.getTableScannerBuilder(tableName, null).getTableScannerBuilderBase64String());
				} catch (Exception e) {
					LOG.error(StringUtils.stringifyException(e));
					throw new RuntimeException(e);
				}
			}		    
		    this.conf = conf;
		  }

	@Override
	public Configuration getConf() {
		return conf;
	}

	@Override
	public List<InputSplit> getSplits(JobContext context) throws IOException,
			InterruptedException {
		if (LOG.isDebugEnabled())
			SpliceLogUtils.debug(LOG, "getSplits");
		TableInputFormat tableInputFormat = new TableInputFormat();
		conf.set(TableInputFormat.INPUT_TABLE,conf.get(MRConstants.SPLICE_INPUT_CONGLOMERATE));		
		tableInputFormat.setConf(conf);
		try {
			tableInputFormat.setScan(TableScannerBuilder.getTableScannerBuilderFromBase64String(conf.get(MRConstants.SPLICE_SCAN_INFO)).getScan());
		} catch (StandardException e) {			
			throw new IOException(e);
		}
		List<InputSplit> splits = tableInputFormat.getSplits(context);
		if (LOG.isTraceEnabled()) {
			SpliceLogUtils.trace(LOG, "getSplits");
			for (InputSplit split: splits) {
				SpliceLogUtils.trace(LOG, "split -> " + split);				
			}
		}
		return splits;
{'fraction_non_alphanumeric': 0.10554252561412171, 'fraction_numerical': 0.0012344155042587335, 'mean_word_length': 4.481732070365359, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7061643', 'n_tokens_mistral': 2865, 'n_tokens_neox': 2497, 'n_words': 463}