<commit_before>package com.splicemachine.derby.utils.marshall;

import com.splicemachine.encoding.MultiFieldDecoder;
import org.apache.derby.iapi.error.StandardException;
import org.apache.derby.iapi.sql.execute.ExecRow;
import org.apache.derby.iapi.types.DataValueDescriptor;
import org.apache.hadoop.hbase.KeyValue;

/**
 * @author <NAME>
 * Created on: 6/12/13
 */
public class RowDecoder {
    private final KeyMarshall keyType;
    /* a mapping from HBase column position -> ExecRow position */
    private final int[] reversedKeyColumns;

    private MultiFieldDecoder keyDecoder;

    private final RowMarshall rowType;
    /* a mapping from HBase column position -> ExecRow position */
    private final int[] reversedRowColumns;
    private boolean clone;
    private final boolean[] sortOrder;

    private MultiFieldDecoder rowDecoder;

    private final ExecRow template;

    public static RowDecoder create(ExecRow template,
                                    KeyMarshall keyType,
                                    int[] keyColumns,
                                    boolean[] sortOrder,
                                    RowMarshall rowType,
                                    int[] rowColumns,
                                    boolean clone) {
        return new RowDecoder(template,keyType, keyColumns, sortOrder,rowType,rowColumns,clone);
    }

    RowDecoder(ExecRow template,
                       KeyMarshall keyType,
                       int[] reversedKeyColumns,
                       boolean[] sortOrder,
                       RowMarshall rowType,
                       int[] reversedRowColumns,
                       boolean clone) {
        this.keyType = keyType;
        this.rowType = rowType;
        this.template = template;
        this.sortOrder = sortOrder;
        this.reversedKeyColumns = reversedKeyColumns;
        this.reversedRowColumns = reversedRowColumns;
        this.clone = clone;

        if(reversedKeyColumns!=null && reversedKeyColumns.length>0){
            //some columns are stored in the row key, so need to initialize
            //a decoder for that
            keyDecoder = MultiFieldDecoder.create();
        }

        if(!rowType.isColumnar()){
            //our data is in a single column
            rowDecoder = MultiFieldDecoder.create();
        }
    }

    public ExecRow decode(Iterable<? extends KeyValue> keyValues) throws StandardException {
        ExecRow row = clone?template.getClone(): template;
        boolean rowSet = false;
        DataValueDescriptor[] fields = row.getRowArray();
        for(KeyValue value: keyValues){
            if(!rowSet){
                if(keyDecoder==null)
                    rowSet=true;
                else{
                    keyDecoder.set(value.getRow());
                    rowSet=true;
                }
            }
            rowType.decode(value,fields,reversedKeyColumns,rowDecoder);
        }

        keyType.decode(fields, reversedKeyColumns, sortOrder,keyDecoder);
        return row;
    }

    public ExecRow decode(KeyValue[] keyValues) throws StandardException{
        boolean rowSet = keyType == null; //if keyType!=null, we want to set on the keyDecoder, otherwise don't bother
        DataValueDescriptor []fields = template.getRowArray();
        for(KeyValue value: keyValues){
            if(!rowSet){
                if(keyDecoder==null)
                    rowSet=true;
                else{
                    keyDecoder.set(value.getRow());
                    rowSet=true;
                }
            }
            rowType.decode(value,fields,reversedRowColumns,rowDecoder);
        }

        if(keyType!=null)
            keyType.decode(fields, reversedKeyColumns,sortOrder, keyDecoder);
        return template;
    }


    public int[] getKeyColumns() {
        return reversedKeyColumns;
    }

    public void reset() {
        if(keyDecoder!=null)
            keyDecoder.reset();
        if(rowDecoder!=null)
            rowDecoder.reset();
    }

    public ExecRow getTemplate() {
        return template;
    }
}
<commit_msg>Resets rows to all null in between decodings<commit_after>package com.splicemachine.derby.utils.marshall;

import com.splicemachine.encoding.MultiFieldDecoder;
import org.apache.derby.iapi.error.StandardException;
import org.apache.derby.iapi.sql.execute.ExecRow;
import org.apache.derby.iapi.types.DataValueDescriptor;
import org.apache.hadoop.hbase.KeyValue;

/**
 * @author <NAME>
 * Created on: 6/12/13
 */
public class RowDecoder {
    private final KeyMarshall keyType;
    /* a mapping from HBase column position -> ExecRow position */
    private final int[] reversedKeyColumns;

    private MultiFieldDecoder keyDecoder;

    private final RowMarshall rowType;
    /* a mapping from HBase column position -> ExecRow position */
    private final int[] reversedRowColumns;
    private boolean clone;
    private final boolean[] sortOrder;

    private MultiFieldDecoder rowDecoder;

    private final ExecRow template;

    public static RowDecoder create(ExecRow template,
                                    KeyMarshall keyType,
                                    int[] keyColumns,
                                    boolean[] sortOrder,
                                    RowMarshall rowType,
                                    int[] rowColumns,
                                    boolean clone) {
        return new RowDecoder(template,keyType, keyColumns, sortOrder,rowType,rowColumns,clone);
    }

    RowDecoder(ExecRow template,
                       KeyMarshall keyType,
                       int[] reversedKeyColumns,
                       boolean[] sortOrder,
                       RowMarshall rowType,
                       int[] reversedRowColumns,
                       boolean clone) {
        this.keyType = keyType;
        this.rowType = rowType;
        this.template = template;
        this.sortOrder = sortOrder;
        this.reversedKeyColumns = reversedKeyColumns;
        this.reversedRowColumns = reversedRowColumns;
        this.clone = clone;

        if(reversedKeyColumns!=null && reversedKeyColumns.length>0){
            //some columns are stored in the row key, so need to initialize
            //a decoder for that
            keyDecoder = MultiFieldDecoder.create();
        }

        if(!rowType.isColumnar()){
            //our data is in a single column
            rowDecoder = MultiFieldDecoder.create();
        }
    }

    public ExecRow decode(Iterable<? extends KeyValue> keyValues) throws StandardException {
        ExecRow row = clone?template.getClone(): template;
        boolean rowSet = false;
        DataValueDescriptor[] fields = row.getRowArray();
        for(KeyValue value: keyValues){
            if(!rowSet){
                if(keyDecoder==null)
                    rowSet=true;
                else{
                    keyDecoder.set(value.getRow());
                    rowSet=true;
                }
            }
            rowType.decode(value,fields,reversedKeyColumns,rowDecoder);
        }

        keyType.decode(fields, reversedKeyColumns, sortOrder,keyDecoder);
        return row;
    }

    public ExecRow decode(KeyValue[] keyValues) throws StandardException{
        boolean rowSet = keyType == null; //if keyType!=null, we want to set on the keyDecoder, otherwise don't bother
        DataValueDescriptor []fields = template.getRowArray();
        for(DataValueDescriptor field:fields){
            field.setToNull(); //unset the value, so that previous runs don't break
        }
        for(KeyValue value: keyValues){
            if(!rowSet){
                if(keyDecoder==null)
                    rowSet=true;
                else{
                    keyDecoder.set(value.getRow());
                    rowSet=true;
                }
            }
            rowType.decode(value,fields,reversedRowColumns,rowDecoder);
        }

        if(keyType!=null)
            keyType.decode(fields, reversedKeyColumns,sortOrder, keyDecoder);
        return template;
    }


    public int[] getKeyColumns() {
        return reversedKeyColumns;
    }

    public void reset() {
        if(keyDecoder!=null)
            keyDecoder.reset();
        if(rowDecoder!=null)
            rowDecoder.reset();
    }

    public ExecRow getTemplate() {
        return template;
    }
}

{'fraction_non_alphanumeric': 0.08319407973263308, 'fraction_numerical': 0.0014323227500596801, 'mean_word_length': 1.8403389830508474, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3876941', 'n_tokens_mistral': 2167, 'n_tokens_neox': 2033, 'n_words': 499}