<commit_before>package water.fvec;

import water.H2O;
import water.util.UnsafeUtils;

import java.util.Arrays;
import java.util.Iterator;

// Sparse chunk.
public class CXIChunk extends Chunk {
  private transient int _valsz; // byte size of stored value
  protected final int valsz() { return _valsz; }
  private transient int _valsz_log; //
  protected transient int _ridsz; // byte size of stored (chunk-relative) row nums
  protected final int ridsz() { return _ridsz; }
  protected transient int _sparseLen;
  protected static final int _OFF = 6;
  private transient int _lastOff = _OFF;

  private static final long [] NAS = {C1Chunk._NA,C2Chunk._NA,C4Chunk._NA,C8Chunk._NA};

  protected CXIChunk(int len, int valsz, byte [] buf){
    assert (valsz == 0 || valsz == 1 || valsz == 2 || valsz == 4 || valsz == 8);
    set_len(len);
    int log = 0;
    while((1 << log) < valsz)++log;
    assert valsz == 0 || (1 << log) == valsz;
    _valsz = valsz;
    _valsz_log = log;
    _ridsz = (len >= 65535)?4:2;
    UnsafeUtils.set4(buf, 0, len);
    byte b = (byte) _ridsz;
    buf[4] = b;
    buf[5] = (byte) _valsz;
    _mem = buf;
    _sparseLen = (_mem.length - _OFF) / (_valsz+_ridsz);
    assert (_mem.length - _OFF) % (_valsz+_ridsz) == 0:"unexpected mem buffer length: mem.length = " + _mem.length + ", off = " + _OFF + ", valSz = " + _valsz + "ridsz = " + _ridsz;
  }

  @Override public double [] getDoubles(double [] vals,int from, int to){
    double fill = isSparseNA()?Double.NaN:0;
    if(from == 0 && to == _len) {
      Arrays.fill(vals,fill);
      double [] svals = new double[_sparseLen];
      int [] sids = new int[_sparseLen];
      asSparseDoubles(svals,sids);
      int start = 0;
      while(start < sids.length && sids[start] < from)++start;
      for(int i = start; i < sids.length && sids[i] < to; ++i)
        vals[sids[i]] = svals[i];
    } else {
      for(int i = from; i < to; ++i)
        vals[i-from] = fill;
      for(int i= nextNZ(from-1); i < to; i = nextNZ(i))
        vals[i-from] = atd(i);
    }
    return vals;
  }

  @Override public int asSparseDoubles(double [] vals, int[] ids) {
    if(vals.length < _sparseLen)throw new IllegalArgumentException();
    int off = _OFF;
    final int inc = _valsz + _ridsz;
    if(_ridsz == 2){
      switch(_valsz){
        case 1:
          for (int i = 0; i < _sparseLen; ++i, off += inc) {
            ids[i] = UnsafeUtils.get2(_mem,off);
            vals[i] = _mem[off+2]&0xFF;
          }
          break;
        case 2:
          for (int i = 0; i < _sparseLen; ++i, off += inc) {
            ids[i] = UnsafeUtils.get2(_mem,off);
            vals[i] = UnsafeUtils.get2(_mem,off+2);
          }
          break;
        case 4:
          for (int i = 0; i < _sparseLen; ++i, off += inc) {
            ids[i] = UnsafeUtils.get2(_mem,off);
            vals[i] = UnsafeUtils.get4(_mem,off+2);
          }
          break;
        case 8:
          for (int i = 0; i < _sparseLen; ++i, off += inc) {
            ids[i] = UnsafeUtils.get2(_mem,off);
            vals[i] = UnsafeUtils.get8(_mem,off+2);
          }
          break;
      }
    } else if(_ridsz == 4){
      switch(_valsz){
        case 1:
          for (int i = 0; i < _sparseLen; ++i, off += inc) {
            ids[i] = UnsafeUtils.get4(_mem,off);
            vals[i] = _mem[off+4]&0xFF;
          }
          break;
        case 2:
          for (int i = 0; i < _sparseLen; ++i, off += inc) {
            ids[i] = UnsafeUtils.get4(_mem,off);
            vals[i] = UnsafeUtils.get2(_mem,off+4);
          }
          break;
        case 4:
          for (int i = 0; i < _sparseLen; ++i, off += inc) {
            ids[i] = UnsafeUtils.get4(_mem,off);
            vals[i] = UnsafeUtils.get4(_mem,off+4);
          }
          break;
        case 8:
          for (int i = 0; i < _sparseLen; ++i, off += inc) {
            ids[i] = UnsafeUtils.get4(_mem,off);
            vals[i] = UnsafeUtils.get8(_mem,off+4);
          }
          break;
      }
    } else throw H2O.unimpl();
    return sparseLenZero();
  }

  @Override public boolean isSparseZero() {return true;}
  @Override public int sparseLenZero(){ return _sparseLen; }
  @Override public int nextNZ(int rid){
    final int off = rid == -1?_OFF:findOffset(rid);
    int x = getId(off);
    if(x > rid)return x;
    if(off < _mem.length - _ridsz - _valsz)
      return getId(off + _ridsz + _valsz);
    return _len;
  }
  /** Fills in a provided (recycled/reused) temp array of the NZ indices, and
   *  returns the count of them.  Array must be large enough. */
  @Override public int nonzeros(int [] arr){
    int off = _OFF;
    final int inc = _valsz + _ridsz;
    for(int i = 0; i < _sparseLen; ++i, off += inc) arr[i] = getId(off);
    return _sparseLen;
  }
  
  @Override boolean set_impl(int idx, long l)   { return false; }<commit_msg>Quick fix for CXIChunk NA handling, might be faster to do it everywhere in-place.<commit_after>package water.fvec;

import water.H2O;
import water.util.UnsafeUtils;

import java.util.Arrays;
import java.util.Iterator;

// Sparse chunk.
public class CXIChunk extends Chunk {
  private transient int _valsz; // byte size of stored value
  protected final int valsz() { return _valsz; }
  private transient int _valsz_log; //
  protected transient int _ridsz; // byte size of stored (chunk-relative) row nums
  protected final int ridsz() { return _ridsz; }
  protected transient int _sparseLen;
  protected static final int _OFF = 6;
  private transient int _lastOff = _OFF;

  private static final long [] NAS = {C1Chunk._NA,C2Chunk._NA,C4Chunk._NA,C8Chunk._NA};

  protected CXIChunk(int len, int valsz, byte [] buf){
    assert (valsz == 0 || valsz == 1 || valsz == 2 || valsz == 4 || valsz == 8);
    set_len(len);
    int log = 0;
    while((1 << log) < valsz)++log;
    assert valsz == 0 || (1 << log) == valsz;
    _valsz = valsz;
    _valsz_log = log;
    _ridsz = (len >= 65535)?4:2;
    UnsafeUtils.set4(buf, 0, len);
    byte b = (byte) _ridsz;
    buf[4] = b;
    buf[5] = (byte) _valsz;
    _mem = buf;
    _sparseLen = (_mem.length - _OFF) / (_valsz+_ridsz);
    assert (_mem.length - _OFF) % (_valsz+_ridsz) == 0:"unexpected mem buffer length: mem.length = " + _mem.length + ", off = " + _OFF + ", valSz = " + _valsz + "ridsz = " + _ridsz;
  }

  @Override public double [] getDoubles(double [] vals,int from, int to){
    double fill = isSparseNA()?Double.NaN:0;
    if(from == 0 && to == _len) {
      Arrays.fill(vals,fill);
      double [] svals = new double[_sparseLen];
      int [] sids = new int[_sparseLen];
      asSparseDoubles(svals,sids);
      int start = 0;
      while(start < sids.length && sids[start] < from)++start;
      for(int i = start; i < sids.length && sids[i] < to; ++i)
        vals[sids[i]] = svals[i];
    } else {
      for(int i = from; i < to; ++i)
        vals[i-from] = fill;
      for(int i= nextNZ(from-1); i < to; i = nextNZ(i))
        vals[i-from] = atd(i);
    }
    return vals;
  }

  @Override public int asSparseDoubles(double [] vals, int[] ids) {
    if(vals.length < _sparseLen)throw new IllegalArgumentException();
    int off = _OFF;
    final int inc = _valsz + _ridsz;
    if(_ridsz == 2){
      switch(_valsz){
        case 1:
          for (int i = 0; i < _sparseLen; ++i, off += inc) {
            ids[i] = UnsafeUtils.get2(_mem,off);
            vals[i] = _mem[off+2]&0xFF;
          }
          break;
        case 2:
          for (int i = 0; i < _sparseLen; ++i, off += inc) {
            ids[i] = UnsafeUtils.get2(_mem,off);
            vals[i] = UnsafeUtils.get2(_mem,off+2);
          }
          break;
        case 4:
          for (int i = 0; i < _sparseLen; ++i, off += inc) {
            ids[i] = UnsafeUtils.get2(_mem,off);
            vals[i] = UnsafeUtils.get4(_mem,off+2);
          }
          break;
        case 8:
          for (int i = 0; i < _sparseLen; ++i, off += inc) {
            ids[i] = UnsafeUtils.get2(_mem,off);
            vals[i] = UnsafeUtils.get8(_mem,off+2);
          }
          break;
      }
    } else if(_ridsz == 4){
      switch(_valsz){
        case 1:
          for (int i = 0; i < _sparseLen; ++i, off += inc) {
            ids[i] = UnsafeUtils.get4(_mem,off);
            vals[i] = _mem[off+4]&0xFF;
          }
          break;
        case 2:
          for (int i = 0; i < _sparseLen; ++i, off += inc) {
            ids[i] = UnsafeUtils.get4(_mem,off);
            vals[i] = UnsafeUtils.get2(_mem,off+4);
          }
          break;
        case 4:
          for (int i = 0; i < _sparseLen; ++i, off += inc) {
            ids[i] = UnsafeUtils.get4(_mem,off);
            vals[i] = UnsafeUtils.get4(_mem,off+4);
          }
          break;
        case 8:
          for (int i = 0; i < _sparseLen; ++i, off += inc) {
            ids[i] = UnsafeUtils.get4(_mem,off);
            vals[i] = UnsafeUtils.get8(_mem,off+4);
          }
          break;
      }
    } else throw H2O.unimpl();
    for (int i = 0; i < _sparseLen; ++i, off += inc) {
      vals[i] = vals[i] == NAS[_valsz_log] ? Double.NaN : vals[i];
    }
    return sparseLenZero();
  }

  @Override public boolean isSparseZero() {return true;}
  @Override public int sparseLenZero(){ return _sparseLen; }
  @Override public int nextNZ(int rid){
    final int off = rid == -1?_OFF:findOffset(rid);
    int x = getId(off);
    if(x > rid)return x;
    if(off < _mem.length - _ridsz - _valsz)
      return getId(off + _ridsz + _valsz);
    return _len;
  }
  /** Fills in a provided (recycled/reused) temp array of the NZ indices, and
   *  returns the count of them.  Array must be large enough. */
  @Override public int nonzeros(int [] arr){
    int off = _OFF;
    final int inc = _valsz + _ridsz;
    for(int i = 0; i < _sparseLen; ++i, off += inc) arr[i] = getId(off);
    return _sparseLen;
  }
  
  @Override boolean set_impl(int idx, long l)   { return false; }
{'fraction_non_alphanumeric': 0.14717171717171718, 'fraction_numerical': 0.015454545454545455, 'mean_word_length': 2.202134540750323, 'pattern_counts': {'":': 0, '<': 48, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2551218', 'n_tokens_mistral': 3741, 'n_tokens_neox': 3595, 'n_words': 947}