<commit_before>package com.splicemachine.hbase.filter;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterBase;
import org.apache.hadoop.hbase.filter.ParseFilter;
import org.apache.hadoop.hbase.util.Bytes;

import com.google.common.base.Preconditions;

/**
 * This filter is used to filter rows based on whether or not the column key exists
 * (for IS NULL or IS NOT NULL filtering). Since HBase only stores non-null values, 
 * checking whether or not column exists is enough for the filtering.
 * It takes a CompareFilter.CompareOp operator (equal, not equal), a column family 
 * and a column qualifier
 * 
 * @author jessiezhang
 */

public class ColumnNullableFilter extends FilterBase {
	static final Log LOG = LogFactory.getLog(ColumnNullableFilter.class);

	protected byte [] columnFamily;
	protected byte [] columnQualifier;
	private CompareOp compareOp;
	private boolean foundColumn = false;
	private boolean filterIfMissing = false;

	/**
	 * Writable constructor, do not use.
	 */
	public ColumnNullableFilter() {
	}

	/**
	 * Constructor for binary compare of the value of a single column.  If the
	 * column is found and the condition passes, all columns of the row will be
	 * emitted.  If the condition fails, the row will not be emitted.
	 * <p>
	 * Use the filterIfMissing flag to set whether the rest of the columns
	 * in a row will be emitted if the specified column to check is not found in
	 * the row.
	 *
	 * @param family name of column family
	 * @param qualifier name of column qualifier
	 * @param compareOp operator
	 */
	public ColumnNullableFilter(final byte [] family, final byte [] qualifier, final CompareOp compareOp) {
		this.columnFamily = family;
		this.columnQualifier = qualifier;
		this.compareOp = compareOp;
		if (CompareFilter.CompareOp.EQUAL.equals(compareOp))  //is null
			this.filterIfMissing = false;
		else //IS NOT NULL
			this.filterIfMissing = true; 
	}

	/**
	 * @return operator
	 */
	public CompareOp getOperator() {
		return compareOp;
	}

	/**
	 * @return the family
	 */
	public byte[] getFamily() {
		return columnFamily;
	}

	/**
	 * @return the qualifier
	 */
	public byte[] getQualifier() {
		return columnQualifier;
	}

	/**
	 * Get whether entire row should be filtered if column is not found.
	 * @return true if row should be skipped if column not found, false if row
	 * should be let through anyways
	 */
	public boolean getFilterIfMissing() {
		return filterIfMissing;
	}

	/**
	 * Set whether entire row should be filtered if column is not found.
	 * <p>
	 * If true, the entire row will be skipped if the column is not found.
	 * <p>
	 * If false, the row will pass if the column is not found.  This is default.
	 * @param filterIfMissing flag
	 */
	public void setFilterIfMissing(boolean filterIfMissing) {
		this.filterIfMissing = filterIfMissing;
	}

	public ReturnCode filterKeyValue(KeyValue keyValue) {
        if (this.foundColumn)
			return ReturnCode.NEXT_ROW;

        if (!keyValue.matchingColumn(this.columnFamily, this.columnQualifier)) {
            this.foundColumn = false;
            return ReturnCode.INCLUDE;
        }

        this.foundColumn = keyValue.getValue().length > 0;
        return ReturnCode.INCLUDE;<commit_msg>Modified filterKeyValue() so that key value is always returned - we always want the key value, it's up to filterRow() to determine if the row should be returned by checking if the column value mathced.<commit_after>package com.splicemachine.hbase.filter;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.util.ArrayList;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterBase;
import org.apache.hadoop.hbase.filter.ParseFilter;
import org.apache.hadoop.hbase.util.Bytes;

import com.google.common.base.Preconditions;

/**
 * This filter is used to filter rows based on whether or not the column key exists
 * (for IS NULL or IS NOT NULL filtering). Since HBase only stores non-null values, 
 * checking whether or not column exists is enough for the filtering.
 * It takes a CompareFilter.CompareOp operator (equal, not equal), a column family 
 * and a column qualifier
 * 
 * @author jessiezhang
 */

public class ColumnNullableFilter extends FilterBase {
	static final Log LOG = LogFactory.getLog(ColumnNullableFilter.class);

	protected byte [] columnFamily;
	protected byte [] columnQualifier;
	private CompareOp compareOp;
	private boolean foundColumn = false;
	private boolean filterIfMissing = false;

	/**
	 * Writable constructor, do not use.
	 */
	public ColumnNullableFilter() {
	}

	/**
	 * Constructor for binary compare of the value of a single column.  If the
	 * column is found and the condition passes, all columns of the row will be
	 * emitted.  If the condition fails, the row will not be emitted.
	 * <p>
	 * Use the filterIfMissing flag to set whether the rest of the columns
	 * in a row will be emitted if the specified column to check is not found in
	 * the row.
	 *
	 * @param family name of column family
	 * @param qualifier name of column qualifier
	 * @param compareOp operator
	 */
	public ColumnNullableFilter(final byte [] family, final byte [] qualifier, final CompareOp compareOp) {
		this.columnFamily = family;
		this.columnQualifier = qualifier;
		this.compareOp = compareOp;
		if (CompareFilter.CompareOp.EQUAL.equals(compareOp))  //is null
			this.filterIfMissing = false;
		else //IS NOT NULL
			this.filterIfMissing = true; 
	}

	/**
	 * @return operator
	 */
	public CompareOp getOperator() {
		return compareOp;
	}

	/**
	 * @return the family
	 */
	public byte[] getFamily() {
		return columnFamily;
	}

	/**
	 * @return the qualifier
	 */
	public byte[] getQualifier() {
		return columnQualifier;
	}

	/**
	 * Get whether entire row should be filtered if column is not found.
	 * @return true if row should be skipped if column not found, false if row
	 * should be let through anyways
	 */
	public boolean getFilterIfMissing() {
		return filterIfMissing;
	}

	/**
	 * Set whether entire row should be filtered if column is not found.
	 * <p>
	 * If true, the entire row will be skipped if the column is not found.
	 * <p>
	 * If false, the row will pass if the column is not found.  This is default.
	 * @param filterIfMissing flag
	 */
	public void setFilterIfMissing(boolean filterIfMissing) {
		this.filterIfMissing = filterIfMissing;
	}

	public ReturnCode filterKeyValue(KeyValue keyValue) {
        if (this.foundColumn)
			return ReturnCode.INCLUDE;

        if (! keyValue.matchingColumn(this.columnFamily, this.columnQualifier)) {
            this.foundColumn = false;
        } else {
            this.foundColumn = keyValue.getValue().length > 0;
        }

        return ReturnCode.INCLUDE;
{'fraction_non_alphanumeric': 0.08552097428958051, 'fraction_numerical': 0.00027063599458728013, 'mean_word_length': 3.66015132408575, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5456422', 'n_tokens_mistral': 2201, 'n_tokens_neox': 2040, 'n_words': 761}