<commit_before>package water.fvec;

import org.junit.BeforeClass;
import org.junit.Test;
import water.TestUtil;
import water.util.ArrayUtils;
import water.util.Log;
import water.util.PrettyPrint;

public class ChunkSpeedTest extends TestUtil {
  @BeforeClass() public static void setup() { stall_till_cloudsize(1); }

  @Test
  public void run() {
    final int cols = 100;
    final int rows = 100000;
    final int rep = 10;
    double[][] raw = new double[cols][rows];
    for (int j=0; j<cols; ++j) {
      for (int i = 0; i < rows; ++i) {
//        switch (j%2) { //just do 1/2 byte chunks
        switch (j%3) { // do all 3 chunk types
          case 0:
            raw[j][i] = i % 200; //C1NChunk - 1 byte integer
            break;
          case 1:
            raw[j][i] = i % 500; //C2Chunk - 2 byte integer
            break;
          case 2:
            raw[j][i] = i == 17 ? 1 : 0; //CX0Chunk - sparse
            break;
        }
      }
    }
    Chunk[] chunks = new Chunk[cols];
    for (int j=0; j<cols; ++j) {
      chunks[j] = new NewChunk(raw[j]).compress();
      Log.info("Column " + j + " compressed into: " + chunks[j].getClass().toString());
    }

    // raw data
    {
      long start = 0;
      double sum[] = new double[cols];
      for (int r = 0; r < rep; ++r) {
        if (r==rep/10)
          start = System.currentTimeMillis();
        for (int j=0; j<cols; ++j) {
          for (int i = 0; i < rows; ++i) {<commit_msg>Add 3rd option with only 1 chunk type to show the "issue".<commit_after>package water.fvec;

import org.junit.BeforeClass;
import org.junit.Test;
import water.TestUtil;
import water.util.ArrayUtils;
import water.util.Log;
import water.util.PrettyPrint;

public class ChunkSpeedTest extends TestUtil {
  @BeforeClass() public static void setup() { stall_till_cloudsize(1); }

  @Test
  public void run() {
    final int cols = 100;
    final int rows = 100000;
    final int rep = 10;
    double[][] raw = new double[cols][rows];
    for (int j=0; j<cols; ++j) {
      for (int i = 0; i < rows; ++i) {
//        switch (j%1) { //just do 1 byte chunks
//        switch (j%2) { //just do 1/2 byte chunks
        switch (j%3) { // do all 3 chunk types
          case 0:
            raw[j][i] = i % 200; //C1NChunk - 1 byte integer
            break;
          case 1:
            raw[j][i] = i % 500; //C2Chunk - 2 byte integer
            break;
          case 2:
            raw[j][i] = i == 17 ? 1 : 0; //CX0Chunk - sparse
            break;
        }
      }
    }
    Chunk[] chunks = new Chunk[cols];
    for (int j=0; j<cols; ++j) {
      chunks[j] = new NewChunk(raw[j]).compress();
      Log.info("Column " + j + " compressed into: " + chunks[j].getClass().toString());
    }

    // raw data
    {
      long start = 0;
      double sum[] = new double[cols];
      for (int r = 0; r < rep; ++r) {
        if (r==rep/10)
          start = System.currentTimeMillis();
        for (int j=0; j<cols; ++j) {
          for (int i = 0; i < rows; ++i) {
{'fraction_non_alphanumeric': 0.1519281914893617, 'fraction_numerical': 0.030585106382978722, 'mean_word_length': 1.9471106758080314, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3222190', 'n_tokens_mistral': 1096, 'n_tokens_neox': 1018, 'n_words': 318}