<commit_before>package water.api;

import com.google.gson.JsonObject;
import jsr166y.ForkJoinTask;
import jsr166y.RecursiveAction;
import water.*;
import water.util.Log;
import hex.TypeChange;

import java.util.Arrays;

public class ToEnum extends Request {
  protected final H2OHexKey      _key       = new H2OHexKey(KEY);
  protected final Int            _col_index = new Int(COL_INDEX, -1);
  protected final Bool           _to_enum   = new Bool(TO_ENUM,true,"");

////  @Override protected void registered(RequestServer.API_VERSION version) { super.registered(version); }
//

  public static class CollectIntDomain extends MRTask<CollectIntDomain> {
    final int _col;
    int [] _dom;

    public CollectIntDomain(int col){_col = col;}

    public void map(Key k) {
      Key aryKey = ValueArray.getArrayKey(k);
      ValueArray ary = DKV.get(aryKey).get();
      int [] dom = new int[256];
      int n = 0;
      AutoBuffer ab = new AutoBuffer(DKV.get(k).getBytes());
      int nrows = ab.remaining()/ary._rowsize;
      final ValueArray.Column col = ary._cols[_col];
      for(int i = 0; i < nrows; ++i){
        if(ary.isNA(ab,i,col))continue;
        int val = (int)ary.data(ab,i,col);
        int id = Arrays.binarySearch(dom,0,n,val);
        if(id < 0){
          if(n == dom.length)
            dom = Arrays.copyOf(dom,n+(n >>1));
          id = -id - 1;
          for(int j = n; j > id; --j)dom[j] = dom[j-1];
            dom[id] = val;
          ++n;
        }
      }
      _dom = Arrays.copyOf(dom,n);
    }
    public void reduce(CollectIntDomain other){
      if(_dom == null)_dom = other._dom;
      else if(other._dom != null && !Arrays.equals(_dom,other._dom)){
        // merge sort the domains
        int [] res = new int[_dom.length + other._dom.length];
        Arrays.fill(res,Integer.MAX_VALUE);
        int i = 0, j = 0, k = 0;
        while(i < _dom.length && j < other._dom.length){
          if(_dom[i] < other._dom[j])res[k++] = _dom[i++];
          else if(_dom[i] == other._dom[j]){
            res[k++] = _dom[i++];j++;
          } else
            res[k++] = other._dom[j++];
        }
        assert (i == _dom.length || j == other._dom.length);
        for(int ii = i; ii < _dom.length; ++ii)
          res[k++] = _dom[ii];
        for(int jj = j; jj < other._dom.length; ++jj)
          res[k++] = other._dom[jj];
        _dom = Arrays.copyOf(res,k);<commit_msg>Added test for too many unique elements in int -> enum conversion.<commit_after>package water.api;

import com.google.gson.JsonObject;
import jsr166y.ForkJoinTask;
import jsr166y.RecursiveAction;
import water.*;
import water.parser.*;
import water.util.Log;
import hex.TypeChange;

import java.util.Arrays;

public class ToEnum extends Request {
  protected final H2OHexKey      _key       = new H2OHexKey(KEY);
  protected final Int            _col_index = new Int(COL_INDEX, -1);
  protected final Bool           _to_enum   = new Bool(TO_ENUM,true,"");

////  @Override protected void registered(RequestServer.API_VERSION version) { super.registered(version); }
//

  public static class CollectIntDomain extends MRTask<CollectIntDomain> {
    final int _col;
    int [] _dom;

    public CollectIntDomain(int col){_col = col;}

    public void map(Key k) {
      Key aryKey = ValueArray.getArrayKey(k);
      ValueArray ary = DKV.get(aryKey).get();
      int [] dom = new int[256];
      int n = 0;
      AutoBuffer ab = new AutoBuffer(DKV.get(k).getBytes());
      int nrows = ab.remaining()/ary._rowsize;
      final ValueArray.Column col = ary._cols[_col];
      for(int i = 0; i < nrows; ++i){
        if(ary.isNA(ab,i,col))continue;
        int val = (int)ary.data(ab,i,col);
        int id = Arrays.binarySearch(dom,0,n,val);
        if(id < 0){
          if(n == dom.length){
            if(n == water.parser.Enum.MAX_ENUM_SIZE)throw new RuntimeException("too many unique elements!");
            dom = Arrays.copyOf(dom,Math.min(n+(n >>1),water.parser.Enum.MAX_ENUM_SIZE));
          }
          id = -id - 1;
          for(int j = n; j > id; --j)dom[j] = dom[j-1];
            dom[id] = val;
          ++n;
        }
      }
      _dom = Arrays.copyOf(dom,n);
    }
    public void reduce(CollectIntDomain other){
      if(_dom == null)_dom = other._dom;
      else if(other._dom != null && !Arrays.equals(_dom,other._dom)){
        // merge sort the domains
        int [] res = new int[_dom.length + other._dom.length];
        Arrays.fill(res,Integer.MAX_VALUE);
        int i = 0, j = 0, k = 0;
        while(i < _dom.length && j < other._dom.length){
          if(_dom[i] < other._dom[j])res[k++] = _dom[i++];
          else if(_dom[i] == other._dom[j]){
            res[k++] = _dom[i++];j++;
          } else
            res[k++] = other._dom[j++];
        }
        assert (i == _dom.length || j == other._dom.length);
        for(int ii = i; ii < _dom.length; ++ii)
          res[k++] = _dom[ii];
        for(int jj = j; jj < other._dom.length; ++jj)
          res[k++] = other._dom[jj];
        if(k >= water.parser.Enum.MAX_ENUM_SIZE)throw new RuntimeException("too many unique elements!");
        _dom = Arrays.copyOf(res,k);
{'fraction_non_alphanumeric': 0.15330875218319426, 'fraction_numerical': 0.008538715311469047, 'mean_word_length': 2.3664271717831484, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '912108', 'n_tokens_mistral': 1781, 'n_tokens_neox': 1751, 'n_words': 405}