<commit_before>package water.fvec;

import java.io.IOException;
import java.io.InputStream;
import java.util.*;

import water.*;
import water.H2O.H2OCountedCompleter;
import water.exec.Flow;
import water.fvec.Vec.VectorGroup;

import com.google.common.base.Throwables;

/**
 * A collection of named Vecs.  Essentially an R-like data-frame.  Multiple
 * Frames can reference the same Vecs.  A Frame is a lightweight object, it is
 * meant to be cheaply created and discarded for data munging purposes.
 * E.g. to exclude a Vec from a computation on a Frame, create a new Frame that
 * references all the Vecs but this one.
 */
public class Frame extends Iced {
  public String[] _names;
  Key[] _keys;          // Keys for the vectors
  transient Vec[] _vecs;// The Vectors (transient to avoid network traffic)
  private transient Vec _col0;  // First readable vec; fast access to the VectorGroup's Chunk layout

  public Frame( Frame fr ) { this(fr._names.clone(), fr.vecs().clone()); _col0 = null; }
  public Frame( Vec... vecs ){ this(null,vecs);}
  public Frame( String[] names, Vec[] vecs ) {
    // assert names==null || names.length == vecs.length : "Number of columns does not match to number of cols' names.";
    _names=names;
    _vecs=vecs;
    _keys = new Key[vecs.length];
    for( int i=0; i<vecs.length; i++ ) {
      Key k = _keys[i] = vecs[i]._key;
      if( DKV.get(k)==null )    // If not already in KV, put it there
        DKV.put(k,vecs[i]);
    }
    Vec v0 = anyVec();
    if( v0 == null ) return;
    VectorGroup grp = v0.group();
    for( int i=0; i<vecs.length; i++ )
      assert grp.equals(vecs[i].group());
  }
  public Vec vec(String name){
    Vec [] vecs = vecs();
    for(int i = 0; i < _names.length; ++i)
      if(_names[i].equals(name))return vecs[i];
    return null;
  }
  public Frame subframe(String [] names){
    Vec [] vecs = new Vec[names.length];
    HashMap<String, Integer> map = new HashMap<String, Integer>();
    for(int i = 0; i < _names.length; ++i)map.put(_names[i], i);
    for(int i = 0; i < names.length; ++i)
      if(map.containsKey(names[i]))vecs[i] = _vecs[map.get(names[i])];
      else throw new IllegalArgumentException("Missing column called "+names[i]);
    return new Frame(names,vecs);
  }
  public final Vec[] vecs() {
    if( _vecs != null ) return _vecs;
    // Load all Vec headers; load them all in parallel by spawning F/J tasks.
    final Vec [] vecs = new Vec[_keys.length];
    Futures fs = new Futures();
    for( int i=0; i<_keys.length; i++ ) {
      final int ii = i;
      final Key k = _keys[i];
      H2OCountedCompleter t = new H2OCountedCompleter() {
          // We need higher priority here as there is a danger of deadlock in
          // case of many calls from MRTask2 at once (e.g. frame with many
          // vectors invokes rollup tasks for all vectors in parallel).  Should
          // probably be done in CPS style in the future
          @Override public byte priority(){return H2O.MIN_HI_PRIORITY;}
          @Override public void compute2() {
            vecs[ii] = DKV.get(k).get();
            tryComplete();
          }
        };
      H2O.submitTask(t);
      fs.add(t);
    }
    fs.blockForPending();
    return _vecs = vecs;
  }
  // Force a cache-flush & reload, assuming vec mappings were altered remotely
  public final Vec[] reloadVecs() { _vecs=null; return vecs(); }

  /** Finds the first column with a matching name.  */
  public int find( String name ) {
    for( int i=0; i<_names.length; i++ )
      if( name.equals(_names[i]) )
        return i;
    return -1;
  }

  public int find( Vec vec ) {
    for( int i=0; i<_vecs.length; i++ )
      if( vec.equals(_vecs[i]) )
        return i;
    return -1;
  }

 /** Appends a named column, keeping the last Vec as the response */
  public void add( String name, Vec vec ) {
    assert _vecs.length == 0 || anyVec().group().equals(vec.group());
    final int len = _names.length;
    _names = Arrays.copyOf(_names,len+1);
    _vecs  = Arrays.copyOf(_vecs ,len+1);
    _keys  = Arrays.copyOf(_keys ,len+1);
    _names[len] = name;
    _vecs [len] = vec ;
    _keys [len] = vec._key;
  }

  /** Appends an entire Frame */
  public Frame add( Frame fr ) {
    assert anyVec().group().equals(fr.anyVec().group());
    final int len0=    _names.length;
    final int len1= fr._names.length;
    final int len = len0+len1;
    _names = Arrays.copyOf(_names,len);
    _vecs  = Arrays.copyOf(_vecs ,len);
    _keys  = Arrays.copyOf(_keys ,len);
    System.arraycopy(fr._names,0,_names,len0,len1);
    System.arraycopy(fr._vecs ,0,_vecs ,len0,len1);
    System.arraycopy(fr._keys ,0,_keys ,len0,len1);
    return this;
  }

  /** Removes the first column with a matching name.  */
  public Vec remove( String name ) { return remove(find(name)); }

  /** Removes a numbered column. */
  public Vec [] remove( int [] idxs ) {
    for(int i :idxs)if(i < 0 || i > _vecs.length)
      throw new ArrayIndexOutOfBoundsException();
    Arrays.sort(idxs);
    Vec [] res = new Vec[idxs.length];
    Vec [] rem = new Vec[_vecs.length-idxs.length];
    String [] names = new String[rem.length];
    Key    [] keys  = new Key   [rem.length];
    int j = 0;
    int k = 0;
    int l = 0;
    for(int i = 0; i < _vecs.length; ++i)
      if(j < idxs.length && i == idxs[j]){
        ++j;
        res[k++] = _vecs[i];
      } else {
        rem  [l] = _vecs [i];
        names[l] = _names[i];
        keys [l] = _keys [i];
        ++l;
      }<commit_msg>Disallow dup col names in Frames<commit_after>package water.fvec;

import java.io.IOException;
import java.io.InputStream;
import java.util.*;

import water.*;
import water.H2O.H2OCountedCompleter;
import water.exec.Flow;
import water.fvec.Vec.VectorGroup;

import com.google.common.base.Throwables;

/**
 * A collection of named Vecs.  Essentially an R-like data-frame.  Multiple
 * Frames can reference the same Vecs.  A Frame is a lightweight object, it is
 * meant to be cheaply created and discarded for data munging purposes.
 * E.g. to exclude a Vec from a computation on a Frame, create a new Frame that
 * references all the Vecs but this one.
 */
public class Frame extends Iced {
  public String[] _names;
  Key[] _keys;          // Keys for the vectors
  transient Vec[] _vecs;// The Vectors (transient to avoid network traffic)
  private transient Vec _col0;  // First readable vec; fast access to the VectorGroup's Chunk layout

  public Frame( Frame fr ) { this(fr._names.clone(), fr.vecs().clone()); _col0 = null; }
  public Frame( Vec... vecs ){ this(null,vecs);}
  public Frame( String[] names, Vec[] vecs ) {
    // assert names==null || names.length == vecs.length : "Number of columns does not match to number of cols' names.";
    _names=names;
    _vecs=vecs;
    _keys = new Key[vecs.length];
    for( int i=0; i<vecs.length; i++ ) {
      Key k = _keys[i] = vecs[i]._key;
      if( DKV.get(k)==null )    // If not already in KV, put it there
        DKV.put(k,vecs[i]);
    }
    Vec v0 = anyVec();
    if( v0 == null ) return;
    VectorGroup grp = v0.group();
    for( int i=0; i<vecs.length; i++ )
      assert grp.equals(vecs[i].group());
  }
  public Vec vec(String name){
    Vec [] vecs = vecs();
    for(int i = 0; i < _names.length; ++i)
      if(_names[i].equals(name))return vecs[i];
    return null;
  }
  public Frame subframe(String [] names){
    Vec [] vecs = new Vec[names.length];
    HashMap<String, Integer> map = new HashMap<String, Integer>();
    for(int i = 0; i < _names.length; ++i)map.put(_names[i], i);
    for(int i = 0; i < names.length; ++i)
      if(map.containsKey(names[i]))vecs[i] = _vecs[map.get(names[i])];
      else throw new IllegalArgumentException("Missing column called "+names[i]);
    return new Frame(names,vecs);
  }
  public final Vec[] vecs() {
    if( _vecs != null ) return _vecs;
    // Load all Vec headers; load them all in parallel by spawning F/J tasks.
    final Vec [] vecs = new Vec[_keys.length];
    Futures fs = new Futures();
    for( int i=0; i<_keys.length; i++ ) {
      final int ii = i;
      final Key k = _keys[i];
      H2OCountedCompleter t = new H2OCountedCompleter() {
          // We need higher priority here as there is a danger of deadlock in
          // case of many calls from MRTask2 at once (e.g. frame with many
          // vectors invokes rollup tasks for all vectors in parallel).  Should
          // probably be done in CPS style in the future
          @Override public byte priority(){return H2O.MIN_HI_PRIORITY;}
          @Override public void compute2() {
            vecs[ii] = DKV.get(k).get();
            tryComplete();
          }
        };
      H2O.submitTask(t);
      fs.add(t);
    }
    fs.blockForPending();
    return _vecs = vecs;
  }
  // Force a cache-flush & reload, assuming vec mappings were altered remotely
  public final Vec[] reloadVecs() { _vecs=null; return vecs(); }

  /** Finds the first column with a matching name.  */
  public int find( String name ) {
    for( int i=0; i<_names.length; i++ )
      if( name.equals(_names[i]) )
        return i;
    return -1;
  }

  public int find( Vec vec ) {
    for( int i=0; i<_vecs.length; i++ )
      if( vec.equals(_vecs[i]) )
        return i;
    return -1;
  }

 /** Appends a named column, keeping the last Vec as the response */
  public void add( String name, Vec vec ) {
    assert _vecs.length == 0 || anyVec().group().equals(vec.group());
    if( find(name) != -1 ) throw new IllegalArgumentException("Duplicate name '"+name+"' in Frame");
    final int len = _names.length;
    _names = Arrays.copyOf(_names,len+1);
    _vecs  = Arrays.copyOf(_vecs ,len+1);
    _keys  = Arrays.copyOf(_keys ,len+1);
    _names[len] = name;
    _vecs [len] = vec ;
    _keys [len] = vec._key;
  }

  /** Appends an entire Frame */
  public Frame add( Frame fr ) { return add(fr,fr._names); }
  public Frame add( Frame fr, String names[] ) {
    assert anyVec().group().equals(fr.anyVec().group());
    for( String name : names )
      if( find(name) != -1 ) throw new IllegalArgumentException("Duplicate name '"+name+"' in Frame");
    final int len0= _names.length;
    final int len1=  names.length;
    final int len = len0+len1;
    _names = Arrays.copyOf(_names,len);
    _vecs  = Arrays.copyOf(_vecs ,len);
    _keys  = Arrays.copyOf(_keys ,len);
    System.arraycopy(    names,0,_names,len0,len1);
    System.arraycopy(fr._vecs ,0,_vecs ,len0,len1);
    System.arraycopy(fr._keys ,0,_keys ,len0,len1);
    return this;
  }
  public Frame add( Frame fr, boolean rename ) { 
    if( !rename ) return add(fr);
    String names[] = new String[fr._names.length];
    for( int i=0; i<names.length; i++ ) {
      String name = fr._names[i];
      int cnt=0;
      while( find(name) != -1 )
        name = fr._names[i]+(cnt++);
      names[i] = name;
    }
    return add(fr,names);
  }

  /** Removes the first column with a matching name.  */
  public Vec remove( String name ) { return remove(find(name)); }

  /** Removes a numbered column. */
  public Vec [] remove( int [] idxs ) {
    for(int i :idxs)if(i < 0 || i > _vecs.length)
      throw new ArrayIndexOutOfBoundsException();
    Arrays.sort(idxs);
    Vec [] res = new Vec[idxs.length];
    Vec [] rem = new Vec[_vecs.length-idxs.length];
    String [] names = new String[rem.length];
    Key    [] keys  = new Key   [rem.length];
    int j = 0;
    int k = 0;
    int l = 0;
    for(int i = 0; i < _vecs.length; ++i)
      if(j < idxs.length && i == idxs[j]){
        ++j;
        res[k++] = _vecs[i];
      } else {
        rem  [l] = _vecs [i];
        names[l] = _names[i];
        keys [l] = _keys [i];
        ++l;
      }
{'fraction_non_alphanumeric': 0.14089935760171307, 'fraction_numerical': 0.008137044967880086, 'mean_word_length': 2.6751652502360717, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4608206', 'n_tokens_mistral': 4027, 'n_tokens_neox': 3969, 'n_words': 1233}