<commit_before>package com.splicemachine.derby.impl.store.access.base;

import com.google.common.io.Closeables;
import com.splicemachine.derby.hbase.SpliceDriver;
import com.splicemachine.derby.hbase.SpliceOperationCoprocessor;
import com.splicemachine.derby.impl.sql.execute.LazyScan;
import com.splicemachine.derby.impl.sql.execute.ParallelScan;
import com.splicemachine.derby.impl.store.access.SpliceAccessManager;
import com.splicemachine.derby.impl.store.access.SpliceTransaction;
import com.splicemachine.derby.impl.store.access.hbase.HBaseRowLocation;
import com.splicemachine.derby.utils.DerbyBytesUtil;
import com.splicemachine.derby.utils.EncodingUtils;
import com.splicemachine.derby.utils.Scans;
import com.splicemachine.derby.utils.SpliceUtils;
import com.splicemachine.derby.utils.marshall.RowDecoder;
import com.splicemachine.derby.utils.marshall.RowMarshaller;
import com.splicemachine.storage.EntryDecoder;
import com.splicemachine.storage.EntryEncoder;
import com.splicemachine.utils.SpliceLogUtils;

import org.apache.derby.iapi.error.StandardException;
import org.apache.derby.iapi.services.io.FormatableBitSet;
import org.apache.derby.iapi.store.access.BackingStoreHashtable;
import org.apache.derby.iapi.store.access.Qualifier;
import org.apache.derby.iapi.store.access.RowUtil;
import org.apache.derby.iapi.store.access.ScanInfo;
import org.apache.derby.iapi.store.access.conglomerate.ScanManager;
import org.apache.derby.iapi.store.raw.Transaction;
import org.apache.derby.iapi.types.DataValueDescriptor;
import org.apache.derby.iapi.types.DataValueFactory;
import org.apache.derby.iapi.types.RowLocation;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.log4j.Logger;

import java.io.IOException;

public class SpliceScan implements ScanManager, ParallelScan, LazyScan {
	protected static Logger LOG = Logger.getLogger(SpliceScan.class);
	protected OpenSpliceConglomerate spliceConglomerate;
	protected Transaction trans;
	protected String transID;
	protected Scan scan;
	protected FormatableBitSet scanColumnList;
    protected RowDecoder rowDecoder;
	protected DataValueDescriptor[] startKeyValue;
	protected int startSearchOperator;
	protected Qualifier[][] qualifier;
	protected DataValueDescriptor[] stopKeyValue;
	protected int stopSearchOperator;
	protected ResultScanner scanner;
	protected HTableInterface table;
	protected boolean currentRowDeleted = false;
	protected HBaseRowLocation currentRowLocation;
	protected DataValueDescriptor[] currentRow;
	protected Result currentResult;
	protected long estimatedRowCount = 0;
	protected boolean isKeyed;
	protected boolean scannerInitialized = false;
	protected String tableName;
    private int[] rowColumns;
    private EntryEncoder entryEncoder;
    private EntryDecoder entryDecoder;

    public SpliceScan() {
		if (LOG.isTraceEnabled())
			LOG.trace("Instantiate Splice Scan for conglomerate ");
	}

	public SpliceScan(FormatableBitSet scanColumnList,
			DataValueDescriptor[] startKeyValue,
			int startSearchOperator,
			Qualifier[][] qualifier,
			DataValueDescriptor[] stopKeyValue,
			int stopSearchOperator,
			Transaction trans, 
			boolean isKeyed) {
		this.isKeyed = isKeyed;
		this.scanColumnList = scanColumnList;
		this.startKeyValue = startKeyValue;
		this.startSearchOperator = startSearchOperator;
		this.qualifier = qualifier;
		this.stopKeyValue = stopKeyValue;
		this.stopSearchOperator = stopSearchOperator;
		this.trans = trans;
		try {
			((SpliceTransaction)trans).setActiveState(false, false, false, null);
		} catch (Exception e) {
			e.printStackTrace();
		}
		this.transID = SpliceUtils.getTransID(trans);
		setupScan();
		attachFilter();
		tableName = Bytes.toString(SpliceOperationCoprocessor.TEMP_TABLE);
        this.rowColumns = new int[scanColumnList.size()];
        int pos=0;
        for(int i=scanColumnList.anySetBit();i!=-1;i=scanColumnList.anySetBit(i)){
            rowColumns[pos] = i;
            pos++;
        }
//		table = SpliceAccessManager.getHTable(SpliceOperationCoprocessor.TEMP_TABLE);
	}

	
	public SpliceScan(OpenSpliceConglomerate spliceConglomerate,
			FormatableBitSet scanColumnList,
			DataValueDescriptor[] startKeyValue,
			int startSearchOperator,
			Qualifier[][] qualifier,
			DataValueDescriptor[] stopKeyValue,
			int stopSearchOperator,
			Transaction trans, 
			boolean isKeyed) {
		this.spliceConglomerate = spliceConglomerate;
		this.isKeyed = isKeyed;
		this.scanColumnList = scanColumnList;
		this.startKeyValue = startKeyValue;
		this.startSearchOperator = startSearchOperator;
		this.qualifier = qualifier;
		this.stopKeyValue = stopKeyValue;
		this.stopSearchOperator = stopSearchOperator;
		this.trans = trans;
		this.transID = SpliceUtils.getTransID(trans);
		setupScan();
		attachFilter();
		tableName = spliceConglomerate.getConglomerate().getContainerid() + "";
        if(scanColumnList!=null){
            this.rowColumns = new int[scanColumnList.size()];
            int pos=0;
            for(int i=scanColumnList.anySetBit();i!=-1;i=scanColumnList.anySetBit(i)){
                rowColumns[pos] = i;
                pos++;
            }
        }
	}
	
	public void close() throws StandardException {
        if(entryDecoder!=null)
            entryDecoder.close();
        if(entryEncoder!=null)<commit_msg>DB-1155 Fixed the ordering of the row column mapping by filling in unneeded columns with -1's.<commit_after>package com.splicemachine.derby.impl.store.access.base;

import com.google.common.io.Closeables;
import com.splicemachine.derby.hbase.SpliceDriver;
import com.splicemachine.derby.hbase.SpliceOperationCoprocessor;
import com.splicemachine.derby.impl.sql.execute.LazyScan;
import com.splicemachine.derby.impl.sql.execute.ParallelScan;
import com.splicemachine.derby.impl.store.access.SpliceAccessManager;
import com.splicemachine.derby.impl.store.access.SpliceTransaction;
import com.splicemachine.derby.impl.store.access.hbase.HBaseRowLocation;
import com.splicemachine.derby.utils.DerbyBytesUtil;
import com.splicemachine.derby.utils.EncodingUtils;
import com.splicemachine.derby.utils.Scans;
import com.splicemachine.derby.utils.SpliceUtils;
import com.splicemachine.derby.utils.marshall.RowDecoder;
import com.splicemachine.derby.utils.marshall.RowMarshaller;
import com.splicemachine.storage.EntryDecoder;
import com.splicemachine.storage.EntryEncoder;
import com.splicemachine.utils.SpliceLogUtils;

import org.apache.derby.iapi.error.StandardException;
import org.apache.derby.iapi.services.io.FormatableBitSet;
import org.apache.derby.iapi.store.access.BackingStoreHashtable;
import org.apache.derby.iapi.store.access.Qualifier;
import org.apache.derby.iapi.store.access.RowUtil;
import org.apache.derby.iapi.store.access.ScanInfo;
import org.apache.derby.iapi.store.access.conglomerate.ScanManager;
import org.apache.derby.iapi.store.raw.Transaction;
import org.apache.derby.iapi.types.DataValueDescriptor;
import org.apache.derby.iapi.types.DataValueFactory;
import org.apache.derby.iapi.types.RowLocation;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.log4j.Logger;

import java.io.IOException;

public class SpliceScan implements ScanManager, ParallelScan, LazyScan {
	protected static Logger LOG = Logger.getLogger(SpliceScan.class);
	protected OpenSpliceConglomerate spliceConglomerate;
	protected Transaction trans;
	protected String transID;
	protected Scan scan;
	protected FormatableBitSet scanColumnList;
    protected RowDecoder rowDecoder;
	protected DataValueDescriptor[] startKeyValue;
	protected int startSearchOperator;
	protected Qualifier[][] qualifier;
	protected DataValueDescriptor[] stopKeyValue;
	protected int stopSearchOperator;
	protected ResultScanner scanner;
	protected HTableInterface table;
	protected boolean currentRowDeleted = false;
	protected HBaseRowLocation currentRowLocation;
	protected DataValueDescriptor[] currentRow;
	protected Result currentResult;
	protected long estimatedRowCount = 0;
	protected boolean isKeyed;
	protected boolean scannerInitialized = false;
	protected String tableName;
    private int[] rowColumns;
    private EntryEncoder entryEncoder;
    private EntryDecoder entryDecoder;

    public SpliceScan() {
		if (LOG.isTraceEnabled())
			LOG.trace("Instantiate Splice Scan for conglomerate ");
	}

	public SpliceScan(FormatableBitSet scanColumnList,
			DataValueDescriptor[] startKeyValue,
			int startSearchOperator,
			Qualifier[][] qualifier,
			DataValueDescriptor[] stopKeyValue,
			int stopSearchOperator,
			Transaction trans, 
			boolean isKeyed) {
		this.isKeyed = isKeyed;
		this.scanColumnList = scanColumnList;
		this.startKeyValue = startKeyValue;
		this.startSearchOperator = startSearchOperator;
		this.qualifier = qualifier;
		this.stopKeyValue = stopKeyValue;
		this.stopSearchOperator = stopSearchOperator;
		this.trans = trans;
		try {
			((SpliceTransaction)trans).setActiveState(false, false, false, null);
		} catch (Exception e) {
			e.printStackTrace();
		}
		this.transID = SpliceUtils.getTransID(trans);
		setupScan();
		attachFilter();
		tableName = Bytes.toString(SpliceOperationCoprocessor.TEMP_TABLE);
        setupRowColumns();
//		table = SpliceAccessManager.getHTable(SpliceOperationCoprocessor.TEMP_TABLE);
	}

	public SpliceScan(OpenSpliceConglomerate spliceConglomerate,
			FormatableBitSet scanColumnList,
			DataValueDescriptor[] startKeyValue,
			int startSearchOperator,
			Qualifier[][] qualifier,
			DataValueDescriptor[] stopKeyValue,
			int stopSearchOperator,
			Transaction trans, 
			boolean isKeyed) {
		this.spliceConglomerate = spliceConglomerate;
		this.isKeyed = isKeyed;
		this.scanColumnList = scanColumnList;
		this.startKeyValue = startKeyValue;
		this.startSearchOperator = startSearchOperator;
		this.qualifier = qualifier;
		this.stopKeyValue = stopKeyValue;
		this.stopSearchOperator = stopSearchOperator;
		this.trans = trans;
		this.transID = SpliceUtils.getTransID(trans);
		setupScan();
		attachFilter();
		tableName = spliceConglomerate.getConglomerate().getContainerid() + "";
        setupRowColumns();
	}

	/**
	 * Initialize the row columns for this scanner.  The index of the array is the position of the field in the HBase "record".
	 * The value of the array element is the position in the returned row (ExecRow).  If the field is not present in the returned row,
	 * -1 is set as the value.
	 */
	private void setupRowColumns() {
		if (scanColumnList != null) {
			int size = scanColumnList.size();
            rowColumns = new int[size];
            for (int i=0; i<size; i++) {
                rowColumns[i] = (scanColumnList.isSet(i) ? i : -1);
            }
        }
	}

	public void close() throws StandardException {
        if(entryDecoder!=null)
            entryDecoder.close();
        if(entryEncoder!=null)
{'fraction_non_alphanumeric': 0.09384994973955954, 'fraction_numerical': 0.0014621219044137804, 'mean_word_length': 5.009884678747941, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7665239', 'n_tokens_mistral': 3562, 'n_tokens_neox': 3339, 'n_words': 640}