<commit_before>package com.splicemachine.derby.utils.marshall;

import com.google.common.collect.Lists;
import com.splicemachine.constants.SIConstants;
import com.splicemachine.constants.SpliceConstants;
import com.splicemachine.derby.utils.DerbyBytesUtil;
import com.splicemachine.derby.utils.Exceptions;
import com.splicemachine.encoding.Encoding;
import com.splicemachine.encoding.MultiFieldDecoder;
import com.splicemachine.encoding.MultiFieldEncoder;
import com.splicemachine.hbase.ByteBufferArrayUtils;
import com.splicemachine.storage.EntryDecoder;
import com.splicemachine.storage.index.BitIndex;

import org.apache.derby.iapi.error.StandardException;
import org.apache.derby.iapi.types.DataValueDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;
import org.xerial.snappy.Snappy;

import java.io.IOException;
import java.util.List;

/**
 * @author <NAME>
 * Created on: 6/21/13
 */
public class RowMarshaller {
    public static final byte[] PACKED_COLUMN_KEY = new byte[]{0x00};
    private static final byte[] EMPTY_ROW_COLUMN = Encoding.encode(-101);

    private static KeyValue getPackedKv(DataValueDescriptor[] row,
                                        byte[] rowKey,
                                        int[] rowColumns,
                                        MultiFieldEncoder encoder, boolean encodeEmpty) throws StandardException{
        pack(row,rowColumns,encoder,encodeEmpty);
        return new KeyValue(rowKey,SpliceConstants.DEFAULT_FAMILY_BYTES,PACKED_COLUMN_KEY,encoder.build());
    }

    private static void pack(DataValueDescriptor[] row, int[] rowColumns,MultiFieldEncoder encoder,boolean encodeEmpty) throws StandardException{

        if(rowColumns!=null){
            for(int rowCol:rowColumns){
                DataValueDescriptor dvd = row[rowCol];
                if(dvd==null){
                    if(encodeEmpty)
                        encoder.encodeEmpty();
                }else if(!dvd.isNull())
                    DerbyBytesUtil.encodeInto(encoder,dvd,false);
                else if(encodeEmpty){
                    DerbyBytesUtil.encodeTypedEmpty(encoder,dvd,false,true);
                }
            }
        }else{
            for (DataValueDescriptor dvd : row) {
                if(dvd==null ){
                   if(encodeEmpty)
                       encoder.encodeEmpty();
                }else if(!dvd.isNull())
                    DerbyBytesUtil.encodeInto(encoder,dvd,false);
                else if(encodeEmpty){
                    DerbyBytesUtil.encodeTypedEmpty(encoder,dvd,false,true);
                }
            }
        }
    }

    private static final RowMarshall SPARSE_PACKED = new RowMarshall() {
        @Override
        public void decode(KeyValue value, DataValueDescriptor[] fields, int[] columns,EntryDecoder entryDecoder) throws StandardException {
            if(!ByteBufferArrayUtils.matchingColumn(value, SpliceConstants.DEFAULT_FAMILY_BYTES,PACKED_COLUMN_KEY)) return;
            entryDecoder.set(value.getBuffer(),value.getValueOffset(),value.getValueLength());
            unpack(fields, columns,entryDecoder);
        }

        @Override
        public byte[] encodeRow(DataValueDescriptor[] row, int[] rowColumns, MultiFieldEncoder rowEncoder) throws StandardException {
                pack(row,rowColumns,rowEncoder,false);
                return rowEncoder.build();
        }

        @Override
        public void encodeKeyValues(DataValueDescriptor[] row,
                                    byte[] rowKey,
                                    int[] rowColumns,
                                    MultiFieldEncoder rowEncoder,
                                    List<KeyValue> kvResults) throws StandardException {
            kvResults.add(getPackedKv(row,rowKey,rowColumns,rowEncoder,false));
        }

        @Override
        public void fill(DataValueDescriptor[] row, int[] rowColumns, MultiFieldEncoder encoder) throws StandardException {
            pack(row,rowColumns,encoder,false);
        }

        @Override
        public void decode(KeyValue value, DataValueDescriptor[] fields, int[] reversedKeyColumns, MultiFieldDecoder rowDecoder) throws StandardException {
            //data is packed in the single value
            if(!ByteBufferArrayUtils.matchingColumn(value, SpliceConstants.DEFAULT_FAMILY_BYTES,PACKED_COLUMN_KEY)) return;
            byte[] data = value.getValue();
            unpack(fields, reversedKeyColumns, rowDecoder, data);
        }

        @Override
        public boolean isColumnar() {
            return false;
        }
    };



    private static final RowMarshall PACKED = new RowMarshall() {
        @Override
        public byte[] encodeRow(DataValueDescriptor[] row, int[] rowColumns, MultiFieldEncoder rowEncoder) throws StandardException {
            pack(row,rowColumns,rowEncoder,true);
            return rowEncoder.build();
        }

        @Override
        public void encodeKeyValues(DataValueDescriptor[] row,
                                    byte[] rowKey,
                                    int[] rowColumns,
                                    MultiFieldEncoder rowEncoder,
                                    List<KeyValue> kvResults) throws StandardException {
            kvResults.add(getPackedKv(row,rowKey,rowColumns,rowEncoder,true));
        }

        @Override
        public void fill(DataValueDescriptor[] row, int[] rowColumns, MultiFieldEncoder encoder) throws StandardException {
            pack(row,rowColumns,encoder,true);
        }

        @Override
        public void decode(KeyValue value, DataValueDescriptor[] fields, int[] reversedKeyColumns, MultiFieldDecoder rowDecoder) throws StandardException {
            //data is packed in the single value
            if(!ByteBufferArrayUtils.matchingColumn(value, SpliceConstants.DEFAULT_FAMILY_BYTES,PACKED_COLUMN_KEY)) return;
            byte[] data = value.getValue();
            unpack(fields, reversedKeyColumns, rowDecoder, data);
        }

        @Override
        public void decode(KeyValue value, DataValueDescriptor[] fields, int[] columns, EntryDecoder entryDecoder) throws StandardException {
            if(!ByteBufferArrayUtils.matchingColumn(value,SpliceConstants.DEFAULT_FAMILY_BYTES,PACKED_COLUMN_KEY)) return;
            entryDecoder.set(value.getBuffer(),value.getValueOffset(),value.getValueLength());
            unpack(fields,columns,entryDecoder);
        }

        @Override
        public boolean isColumnar() {
            return false;
        }
    };

    private static void unpack(DataValueDescriptor[] fields, int[] reversedKeyColumns, MultiFieldDecoder rowDecoder, byte[] data) throws StandardException {
        rowDecoder.set(data);
        if(reversedKeyColumns!=null){
            for(int keyCol:reversedKeyColumns){
                DataValueDescriptor dvd = fields[keyCol];
                if (DerbyBytesUtil.isNextFieldNull(rowDecoder,dvd)) {
                    dvd.setToNull();
                    DerbyBytesUtil.skip(rowDecoder,dvd);
                }else
                    DerbyBytesUtil.decodeInto(rowDecoder, dvd);<commit_msg>Swap out two places where byte[] are created vs. utilizing the existing buffer with offsets<commit_after>package com.splicemachine.derby.utils.marshall;

import com.google.common.collect.Lists;
import com.splicemachine.constants.SIConstants;
import com.splicemachine.constants.SpliceConstants;
import com.splicemachine.derby.utils.DerbyBytesUtil;
import com.splicemachine.derby.utils.Exceptions;
import com.splicemachine.encoding.Encoding;
import com.splicemachine.encoding.MultiFieldDecoder;
import com.splicemachine.encoding.MultiFieldEncoder;
import com.splicemachine.hbase.ByteBufferArrayUtils;
import com.splicemachine.storage.EntryDecoder;
import com.splicemachine.storage.index.BitIndex;

import org.apache.derby.iapi.error.StandardException;
import org.apache.derby.iapi.types.DataValueDescriptor;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;
import org.xerial.snappy.Snappy;

import java.io.IOException;
import java.util.List;

/**
 * @author <NAME>
 * Created on: 6/21/13
 */
public class RowMarshaller {
    public static final byte[] PACKED_COLUMN_KEY = new byte[]{0x00};
    private static final byte[] EMPTY_ROW_COLUMN = Encoding.encode(-101);

    private static KeyValue getPackedKv(DataValueDescriptor[] row,
                                        byte[] rowKey,
                                        int[] rowColumns,
                                        MultiFieldEncoder encoder, boolean encodeEmpty) throws StandardException{
        pack(row,rowColumns,encoder,encodeEmpty);
        return new KeyValue(rowKey,SpliceConstants.DEFAULT_FAMILY_BYTES,PACKED_COLUMN_KEY,encoder.build());
    }

    private static void pack(DataValueDescriptor[] row, int[] rowColumns,MultiFieldEncoder encoder,boolean encodeEmpty) throws StandardException{

        if(rowColumns!=null){
            for(int rowCol:rowColumns){
                DataValueDescriptor dvd = row[rowCol];
                if(dvd==null){
                    if(encodeEmpty)
                        encoder.encodeEmpty();
                }else if(!dvd.isNull())
                    DerbyBytesUtil.encodeInto(encoder,dvd,false);
                else if(encodeEmpty){
                    DerbyBytesUtil.encodeTypedEmpty(encoder,dvd,false,true);
                }
            }
        }else{
            for (DataValueDescriptor dvd : row) {
                if(dvd==null ){
                   if(encodeEmpty)
                       encoder.encodeEmpty();
                }else if(!dvd.isNull())
                    DerbyBytesUtil.encodeInto(encoder,dvd,false);
                else if(encodeEmpty){
                    DerbyBytesUtil.encodeTypedEmpty(encoder,dvd,false,true);
                }
            }
        }
    }

    private static final RowMarshall SPARSE_PACKED = new RowMarshall() {
        @Override
        public void decode(KeyValue value, DataValueDescriptor[] fields, int[] columns,EntryDecoder entryDecoder) throws StandardException {
            if(!ByteBufferArrayUtils.matchingColumn(value, SpliceConstants.DEFAULT_FAMILY_BYTES,PACKED_COLUMN_KEY)) return;
            entryDecoder.set(value.getBuffer(),value.getValueOffset(),value.getValueLength());
            unpack(fields, columns,entryDecoder);
        }

        @Override
        public byte[] encodeRow(DataValueDescriptor[] row, int[] rowColumns, MultiFieldEncoder rowEncoder) throws StandardException {
                pack(row,rowColumns,rowEncoder,false);
                return rowEncoder.build();
        }

        @Override
        public void encodeKeyValues(DataValueDescriptor[] row,
                                    byte[] rowKey,
                                    int[] rowColumns,
                                    MultiFieldEncoder rowEncoder,
                                    List<KeyValue> kvResults) throws StandardException {
            kvResults.add(getPackedKv(row,rowKey,rowColumns,rowEncoder,false));
        }

        @Override
        public void fill(DataValueDescriptor[] row, int[] rowColumns, MultiFieldEncoder encoder) throws StandardException {
            pack(row,rowColumns,encoder,false);
        }

        @Override
        public void decode(KeyValue value, DataValueDescriptor[] fields, int[] reversedKeyColumns, MultiFieldDecoder rowDecoder) throws StandardException {
            //data is packed in the single value
            if(!ByteBufferArrayUtils.matchingColumn(value, SpliceConstants.DEFAULT_FAMILY_BYTES,PACKED_COLUMN_KEY)) return;
            unpack(fields, reversedKeyColumns, rowDecoder, value.getBuffer(), value.getValueOffset(), value.getValueLength());
        }

        @Override
        public boolean isColumnar() {
            return false;
        }
    };



    private static final RowMarshall PACKED = new RowMarshall() {
        @Override
        public byte[] encodeRow(DataValueDescriptor[] row, int[] rowColumns, MultiFieldEncoder rowEncoder) throws StandardException {
            pack(row,rowColumns,rowEncoder,true);
            return rowEncoder.build();
        }

        @Override
        public void encodeKeyValues(DataValueDescriptor[] row,
                                    byte[] rowKey,
                                    int[] rowColumns,
                                    MultiFieldEncoder rowEncoder,
                                    List<KeyValue> kvResults) throws StandardException {
            kvResults.add(getPackedKv(row,rowKey,rowColumns,rowEncoder,true));
        }

        @Override
        public void fill(DataValueDescriptor[] row, int[] rowColumns, MultiFieldEncoder encoder) throws StandardException {
            pack(row,rowColumns,encoder,true);
        }

        @Override
        public void decode(KeyValue value, DataValueDescriptor[] fields, int[] reversedKeyColumns, MultiFieldDecoder rowDecoder) throws StandardException {
            //data is packed in the single value
            if(!ByteBufferArrayUtils.matchingColumn(value, SpliceConstants.DEFAULT_FAMILY_BYTES,PACKED_COLUMN_KEY)) return;
            unpack(fields, reversedKeyColumns, rowDecoder, value.getBuffer(),value.getValueOffset(), value.getValueLength());
        }

        @Override
        public void decode(KeyValue value, DataValueDescriptor[] fields, int[] columns, EntryDecoder entryDecoder) throws StandardException {
            if(!ByteBufferArrayUtils.matchingColumn(value,SpliceConstants.DEFAULT_FAMILY_BYTES,PACKED_COLUMN_KEY)) return;
            entryDecoder.set(value.getBuffer(),value.getValueOffset(),value.getValueLength());
            unpack(fields,columns,entryDecoder);
        }

        @Override
        public boolean isColumnar() {
            return false;
        }
    };

    private static void unpack(DataValueDescriptor[] fields, int[] reversedKeyColumns, MultiFieldDecoder rowDecoder, byte[] data, int offset, int length) throws StandardException {
        rowDecoder.set(data, offset, length);
        if(reversedKeyColumns!=null){
            for(int keyCol:reversedKeyColumns){
                DataValueDescriptor dvd = fields[keyCol];
                if (DerbyBytesUtil.isNextFieldNull(rowDecoder,dvd)) {
                    dvd.setToNull();
                    DerbyBytesUtil.skip(rowDecoder,dvd);
                }else
                    DerbyBytesUtil.decodeInto(rowDecoder, dvd);
{'fraction_non_alphanumeric': 0.08669727914467823, 'fraction_numerical': 0.0015077787677335343, 'mean_word_length': 2.4472005669737773, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4337122', 'n_tokens_mistral': 3777, 'n_tokens_neox': 3625, 'n_words': 720}