<commit_before>package ml.shifu.shifu.core.binning;

import java.util.ArrayList;
import java.util.List;

import ml.shifu.shifu.util.QuickSort;

import org.apache.commons.lang.StringUtils;

public class NativeBinning extends AbstractBinning<Double>{

	private List<Double> array;
	private boolean mergeEnabled;
	private final static double EPS = 1e-5;
	
	public NativeBinning(int binningNum, boolean mergeEnabled) {
		super(binningNum);
		this.mergeEnabled = mergeEnabled;
	}

	@Override
	public void addData(String val) {
		String fval = StringUtils.trimToEmpty(val);
        if ( !isMissingVal(fval) ) {
            double dval = 0;
            
            try {
                dval = Double.parseDouble(fval);
            } catch (NumberFormatException e) {
                super.incInvalidValCnt();
                return;
            }
            array.add(dval);
            
        } else {
            super.incMissingValCnt();
        }
	}

	@Override
	public List<Double> getDataBin() {

		QuickSort.sort(array);
		
		int actualBinSize = (int) Math.ceil((double)array.size() / (double)expectedBinningNum);
		int actualBiningNum = this.expectedBinningNum;
		
		List<Double> binBoundary = new ArrayList<Double>();
		binBoundary.add(Double.NEGATIVE_INFINITY);
		
		double prevData = array.get(0);
		int currBinSize = 0;
		int currBinIndex = 0;
		for(int i = 0 ; i < array.size(); i ++){
			
			double currData = array.get(i);
			currBinSize ++;
			if(currBinSize >= actualBinSize) {
				if(currBinIndex == actualBiningNum - 1 && i != array.size() - 1) {
					continue;
				} 
				
				if (i == 0 || (mergeEnabled == true && Math.abs(currData - prevData) > EPS) || mergeEnabled == false) {
					if (i == array.size() - 1) {
                        break;
                    }
					currBinIndex ++;
					currBinSize = 0;
					binBoundary.add(currData);
				}
				
			}
			
			prevData = currData;
		}
		
		binBoundary.set(binBoundary.size() - 1, Double.POSITIVE_INFINITY);
		return binBoundary;
	}	

}<commit_msg>remove positive infinity for native binning<commit_after>package ml.shifu.shifu.core.binning;

import java.util.ArrayList;
import java.util.List;

import ml.shifu.shifu.util.QuickSort;

import org.apache.commons.lang.StringUtils;

public class NativeBinning extends AbstractBinning<Double>{

	private List<Double> array;
	private boolean mergeEnabled;
	private final static double EPS = 1e-5;
	
	public NativeBinning(int binningNum, boolean mergeEnabled) {
		super(binningNum);
		this.mergeEnabled = mergeEnabled;
	}

	@Override
	public void addData(String val) {
		String fval = StringUtils.trimToEmpty(val);
        if ( !isMissingVal(fval) ) {
            double dval = 0;
            
            try {
                dval = Double.parseDouble(fval);
            } catch (NumberFormatException e) {
                super.incInvalidValCnt();
                return;
            }
            array.add(dval);
            
        } else {
            super.incMissingValCnt();
        }
	}

	@Override
	public List<Double> getDataBin() {

		QuickSort.sort(array);
		
		int actualBinSize = (int) Math.ceil((double)array.size() / (double)expectedBinningNum);
		int actualBiningNum = this.expectedBinningNum;
		
		List<Double> binBoundary = new ArrayList<Double>();
		binBoundary.add(Double.NEGATIVE_INFINITY);
		
		double prevData = array.get(0);
		int currBinSize = 0;
		int currBinIndex = 0;
		for(int i = 0 ; i < array.size(); i ++){
			
			double currData = array.get(i);
			currBinSize ++;
			if(currBinSize >= actualBinSize) {
				if(currBinIndex == actualBiningNum - 1 && i != array.size() - 1) {
					continue;
				} 
				
				if (i == 0 || (mergeEnabled == true && Math.abs(currData - prevData) > EPS) || mergeEnabled == false) {
					if (i == array.size() - 1) {
                        break;
                    }
					currBinIndex ++;
					currBinSize = 0;
					binBoundary.add(currData);
				}
				
			}
			
			prevData = currData;
		}
		
		//binBoundary.set(binBoundary.size() - 1, Double.POSITIVE_INFINITY);
		return binBoundary;
	}	

}
{'fraction_non_alphanumeric': 0.11874386653581943, 'fraction_numerical': 0.006378802747791953, 'mean_word_length': 2.4145728643216082, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '935536', 'n_tokens_mistral': 1511, 'n_tokens_neox': 1300, 'n_words': 267}