<commit_before>package hex.kmeans;

import hex.Model;
import hex.ModelBuilder;
import hex.schemas.KMeansV2;
import hex.schemas.ModelBuilderSchema;
import water.*;
import water.H2O.H2OCountedCompleter;
import water.fvec.Chunk;
import water.fvec.Vec;
import water.util.ArrayUtils;
import water.util.Log;
import water.util.RandomUtils;

import java.util.ArrayList;
import java.util.Random;

/**
 * Scalable K-Means++ (KMeans||)<br>
 * http://theory.stanford.edu/~sergei/papers/vldb12-kmpar.pdf<br>
 * http://www.youtube.com/watch?v=cigXAxV3XcY
 */
public class KMeans extends ModelBuilder<KMeansModel,KMeansModel.KMeansParameters,KMeansModel.KMeansOutput> {
  @Override
  public Model.ModelCategory[] can_build() {
    return new Model.ModelCategory[]{
      Model.ModelCategory.Clustering,
    };
  }

  public enum Initialization {
    None, PlusPlus, Furthest
  }

  // Number of categorical columns
  private int _ncats;

  // Number of reinitialization attempts for preventing empty clusters
  transient private int _reinit_attempts;

  // Called from an http request
  public KMeans( KMeansModel.KMeansParameters parms ) { super("K-means",parms); init(false); }

  public ModelBuilderSchema schema() { return new KMeansV2(); }


  /** Start the KMeans training Job on an F/J thread. */
  @Override public Job<KMeansModel> trainModel() {
    return start(new KMeansDriver(), _parms._max_iters);
  }
  
  /** Initialize the ModelBuilder, validating all arguments and preparing the
   *  training frame.  This call is expected to be overridden in the subclasses
   *  and each subclass will start with "super.init();".
   *
   *  Validate K, max_iters and the number of rows.  Precompute the number of
   *  categorical columns. */
  @Override public void init(boolean expensive) {
    super.init(expensive);
    if( _parms._K < 2 || _parms._K > 9999999 ) error("_K", "K must be between 2 and 10 million");
    if( _parms._max_iters < 1 || _parms._max_iters > 999999) error("_max_iters", "must be between 1 and a million");
    if( _train == null ) return; // Nothing more to check
    if( _train.numRows() < _parms._K ) error("_K","Cannot make " + _parms._K + " clusters out of " + _train.numRows() + " rows.");

    for( Vec v : _train.vecs() )
      if( v.isEnum() ) _ncats++;

    // Sort columns, so the categoricals are all up front.  They use a
    // different distance metric than numeric columns.
    Vec vecs[] = _train.vecs();
    int ncats=0, len=vecs.length; // Feature count;
    while( ncats != len ) {
      while( ncats < len && vecs[ncats].isEnum() ) ncats++;
      while( len > 0 && !vecs[len-1].isEnum() ) len--;
      if( ncats < len-1 ) { _train.swap(ncats,len-1); _valid.swap(ncats,len-1); }
    }
    _ncats = ncats;
  }

  // ----------------------
  private class KMeansDriver extends H2OCountedCompleter<KMeansDriver> {

    @Override protected void compute2() {

      KMeansModel model = null;
      try {
        init(true);
        _parms.lock_frames(KMeans.this); // Fetch & read-lock input frames
<commit_msg>tweak iae messages in kmeans<commit_after>package hex.kmeans;

import hex.Model;
import hex.ModelBuilder;
import hex.schemas.KMeansV2;
import hex.schemas.ModelBuilderSchema;
import water.*;
import water.H2O.H2OCountedCompleter;
import water.fvec.Chunk;
import water.fvec.Vec;
import water.util.ArrayUtils;
import water.util.Log;
import water.util.RandomUtils;

import java.util.ArrayList;
import java.util.Random;

/**
 * Scalable K-Means++ (KMeans||)<br>
 * http://theory.stanford.edu/~sergei/papers/vldb12-kmpar.pdf<br>
 * http://www.youtube.com/watch?v=cigXAxV3XcY
 */
public class KMeans extends ModelBuilder<KMeansModel,KMeansModel.KMeansParameters,KMeansModel.KMeansOutput> {
  @Override
  public Model.ModelCategory[] can_build() {
    return new Model.ModelCategory[]{
      Model.ModelCategory.Clustering,
    };
  }

  public enum Initialization {
    None, PlusPlus, Furthest
  }

  // Number of categorical columns
  private int _ncats;

  // Number of reinitialization attempts for preventing empty clusters
  transient private int _reinit_attempts;

  // Called from an http request
  public KMeans( KMeansModel.KMeansParameters parms ) { super("K-means",parms); init(false); }

  public ModelBuilderSchema schema() { return new KMeansV2(); }


  /** Start the KMeans training Job on an F/J thread. */
  @Override public Job<KMeansModel> trainModel() {
    return start(new KMeansDriver(), _parms._max_iters);
  }
  
  /** Initialize the ModelBuilder, validating all arguments and preparing the
   *  training frame.  This call is expected to be overridden in the subclasses
   *  and each subclass will start with "super.init();".
   *
   *  Validate K, max_iters and the number of rows.  Precompute the number of
   *  categorical columns. */
  @Override public void init(boolean expensive) {
    super.init(expensive);
    if( _parms._K < 2 || _parms._K > 9999999 ) error("_K", "K must be between 2 and 10e6");
    if( _parms._max_iters < 1 || _parms._max_iters > 999999) error("_max_iters", " max_iters must be between 1 and 1e6");
    if( _train == null ) return; // Nothing more to check
    if( _train.numRows() < _parms._K ) error("_K","Cannot make " + _parms._K + " clusters out of " + _train.numRows() + " rows.");

    for( Vec v : _train.vecs() )
      if( v.isEnum() ) _ncats++;

    // Sort columns, so the categoricals are all up front.  They use a
    // different distance metric than numeric columns.
    Vec vecs[] = _train.vecs();
    int ncats=0, len=vecs.length; // Feature count;
    while( ncats != len ) {
      while( ncats < len && vecs[ncats].isEnum() ) ncats++;
      while( len > 0 && !vecs[len-1].isEnum() ) len--;
      if( ncats < len-1 ) { _train.swap(ncats,len-1); _valid.swap(ncats,len-1); }
    }
    _ncats = ncats;
  }

  // ----------------------
  private class KMeansDriver extends H2OCountedCompleter<KMeansDriver> {

    @Override protected void compute2() {

      KMeansModel model = null;
      try {
        init(true);
        _parms.lock_frames(KMeans.this); // Fetch & read-lock input frames

{'fraction_non_alphanumeric': 0.12885245901639344, 'fraction_numerical': 0.011639344262295083, 'mean_word_length': 3.3609721229449607, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5928021', 'n_tokens_mistral': 2111, 'n_tokens_neox': 2005, 'n_words': 545}