<commit_before>package water.api;

import water.AutoBuffer;
import water.H2O;
import water.Iced;
import water.IcedWrapper;
import water.util.Log;
import water.util.TwoDimTable;

/**
 * Client-facing Schema of a TwoDimTable
 * Notes:
 * 1) We embed the rowHeaders into the table, extending it by 1 column
 * 2) We store all the data in column-major order
 * 3) We store all the data in String format
 *
 */
public class TwoDimTableBase<I extends TwoDimTable, S extends TwoDimTableBase> extends Schema<I, TwoDimTableBase<I, S>> {
  public static class ColumnSpecsBase extends Schema<Iced, ColumnSpecsBase> {
    @API(help="Column Name", direction=API.Direction.OUTPUT)
    String name;
    @API(help="Column Type", direction=API.Direction.OUTPUT)
    String type;
    @API(help="Column Format (printf)", direction=API.Direction.OUTPUT)
    String format;
    @API(help="Column Description", direction=API.Direction.OUTPUT)
    String description;
  }

  @API(help="Table Name", direction=API.Direction.OUTPUT)
  public String name;

  @API(help="Table Description", direction=API.Direction.OUTPUT)
  public String description;

  @API(help="Column Specification", direction=API.Direction.OUTPUT)
  public ColumnSpecsBase[] columns;

  @API(help="Number of Rows", direction=API.Direction.OUTPUT)
  public int rowcount;

  @API(help="Table Data (col-major)", direction=API.Direction.OUTPUT)
  public IcedWrapper[][] data;

  /**
   * Fill a TwoDimTable Schema from a TwoDimTable
   * @param t TwoDimTable
   * @return TwoDimTableSchema
   */
  @Override public TwoDimTableBase fillFromImpl(TwoDimTable t) {
    name = t.getTableHeader();
    description = t.getTableDescription();
    final int cols = t.getColDim()+1;
    final int rows = t.getRowDim();
    rowcount = rows;
    columns = new ColumnSpecsBase[cols];
    columns[0] = new ColumnSpecsBase();
    columns[0].name = pythonify(t.getColHeaderForRowHeaders());
    columns[0].type = "string"; //Ugly: Should be an Enum in TwoDimTable class
    columns[0].format = "%s";
    columns[0].description = t.getColHeaderForRowHeaders();
    for (int c=1; c<cols; ++c) {
      columns[c] = new ColumnSpecsBase();
      columns[c].name = pythonify(t.getColHeaders()[c - 1]);
      columns[c].type = t.getColTypes()[c-1];
      columns[c].format = t.getColFormats()[c-1];
      columns[c].description = t.getColHeaders()[c-1];
    }
    data = new IcedWrapper[cols][rows];
    data[0] = new IcedWrapper[t.getRowDim()];
    for (int r=0; r<t.getRowDim(); ++r) {
      data[0][r] = new IcedWrapper(t.getRowHeaders()[r]);
    }
    IcedWrapper[][]cellValues = t.getCellValues();
    for (int c=1; c<cols; ++c) {
      data[c] = new IcedWrapper[rows];
      for (int r=0; r<rows; ++r) {
        data[c][r] = cellValues[r][c-1];
      }
    }
    return this;
  }

  /**
   * Turn a description such as "Avg. Training MSE" into a JSON-usable field name "avg_training_mse"
   * @param name
   * @return
   */
  private String pythonify(String name) {
    StringBuilder sb = new StringBuilder();
    String [] modified = name.split("[\\s_]+");
    for (int i=0; i<modified.length; ++i) {<commit_msg>Don't pythonify null.<commit_after>package water.api;

import water.AutoBuffer;
import water.H2O;
import water.Iced;
import water.IcedWrapper;
import water.util.Log;
import water.util.TwoDimTable;

/**
 * Client-facing Schema of a TwoDimTable
 * Notes:
 * 1) We embed the rowHeaders into the table, extending it by 1 column
 * 2) We store all the data in column-major order
 * 3) We store all the data in String format
 *
 */
public class TwoDimTableBase<I extends TwoDimTable, S extends TwoDimTableBase> extends Schema<I, TwoDimTableBase<I, S>> {
  public static class ColumnSpecsBase extends Schema<Iced, ColumnSpecsBase> {
    @API(help="Column Name", direction=API.Direction.OUTPUT)
    String name;
    @API(help="Column Type", direction=API.Direction.OUTPUT)
    String type;
    @API(help="Column Format (printf)", direction=API.Direction.OUTPUT)
    String format;
    @API(help="Column Description", direction=API.Direction.OUTPUT)
    String description;
  }

  @API(help="Table Name", direction=API.Direction.OUTPUT)
  public String name;

  @API(help="Table Description", direction=API.Direction.OUTPUT)
  public String description;

  @API(help="Column Specification", direction=API.Direction.OUTPUT)
  public ColumnSpecsBase[] columns;

  @API(help="Number of Rows", direction=API.Direction.OUTPUT)
  public int rowcount;

  @API(help="Table Data (col-major)", direction=API.Direction.OUTPUT)
  public IcedWrapper[][] data;

  /**
   * Fill a TwoDimTable Schema from a TwoDimTable
   * @param t TwoDimTable
   * @return TwoDimTableSchema
   */
  @Override public TwoDimTableBase fillFromImpl(TwoDimTable t) {
    name = t.getTableHeader();
    description = t.getTableDescription();
    final int cols = t.getColDim()+1;
    final int rows = t.getRowDim();
    rowcount = rows;
    columns = new ColumnSpecsBase[cols];
    columns[0] = new ColumnSpecsBase();
    columns[0].name = pythonify(t.getColHeaderForRowHeaders());
    columns[0].type = "string";
    columns[0].format = "%s";
    columns[0].description = t.getColHeaderForRowHeaders();
    for (int c=1; c<cols; ++c) {
      columns[c] = new ColumnSpecsBase();
      columns[c].name = pythonify(t.getColHeaders()[c - 1]);
      columns[c].type = t.getColTypes()[c-1];
      columns[c].format = t.getColFormats()[c-1];
      columns[c].description = t.getColHeaders()[c-1];
    }
    data = new IcedWrapper[cols][rows];
    data[0] = new IcedWrapper[t.getRowDim()];
    for (int r=0; r<t.getRowDim(); ++r) {
      data[0][r] = new IcedWrapper(t.getRowHeaders()[r]);
    }
    IcedWrapper[][]cellValues = t.getCellValues();
    for (int c=1; c<cols; ++c) {
      data[c] = new IcedWrapper[rows];
      for (int r=0; r<rows; ++r) {
        data[c][r] = cellValues[r][c-1];
      }
    }
    return this;
  }

  /**
   * Turn a description such as "Avg. Training MSE" into a JSON-usable field name "avg_training_mse"
   * @param name
   * @return
   */
  private String pythonify(String name) {
    if (name == null) return null;
    StringBuilder sb = new StringBuilder();
    String [] modified = name.split("[\\s_]+");
    for (int i=0; i<modified.length; ++i) {
{'fraction_non_alphanumeric': 0.14190051020408162, 'fraction_numerical': 0.007334183673469388, 'mean_word_length': 3.7958715596330275, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3134404', 'n_tokens_mistral': 2054, 'n_tokens_neox': 2028, 'n_words': 517}