<commit_before>package water.rapids.ast.prims.advmath;

import water.DKV;
import water.MRTask;
import water.Scope;
import water.Value;
import water.fvec.Chunk;
import water.fvec.Frame;
import water.fvec.Vec;
import water.rapids.Env;
import water.rapids.Val;
import water.rapids.ast.AstPrimitive;
import water.rapids.ast.AstRoot;
import water.rapids.vals.ValNum;
import water.util.FrameUtils;

public class AstSpearman extends AstPrimitive<AstSpearman> {
  @Override
  public int nargs() {
    return 1 + 3; // Frame ID and ID of two numerical vectors to calculate SCC on.
  }

  @Override
  public String[] args() {
    return new String[]{"frame", "first_column", "second_column"};
  }

  @Override
  public Val apply(Env env, Env.StackHelp stk, AstRoot[] asts) {
    final Value value = DKV.get(asts[1].str());
    if (!value.isFrame()) {
      throw new IllegalArgumentException(String.format("The given key '%s' is not a frame.", asts[1]));
    }

    final Frame originalUnsortedFrame = value.get(Frame.class);
    final int vecIdX = originalUnsortedFrame.find(asts[2].exec(env).getStr());
    final int vecIdY = originalUnsortedFrame.find(asts[3].exec(env).getStr());

    Scope.enter();

    Frame sortedX = new Frame(originalUnsortedFrame.vec(vecIdX).makeCopy());
    Scope.track(sortedX);
    Frame sortedY = new Frame(originalUnsortedFrame.vec(vecIdY).makeCopy());
    Scope.track(sortedY);

    if (!sortedX.vec(0).isCategorical()) {
      FrameUtils.labelRows(sortedX, "label");
      sortedX = sortedX.sort(new int[]{0});
      Scope.track(sortedX);

    }

    if (!sortedY.vec(0).isCategorical()) {
      FrameUtils.labelRows(sortedY, "label");
      sortedY = sortedY.sort(new int[]{0});
      Scope.track(sortedY);
    }

    assert sortedX.numRows() == sortedY.numRows();
    final Vec orderX = Vec.makeZero(sortedX.numRows());
    final Vec orderY = Vec.makeZero(sortedY.numRows());

    final Vec xLabel = sortedX.vec("label") == null ? sortedX.vec(0) : sortedX.vec("label");
    final Vec xValue = sortedX.vec(0);
    final Vec yLabel = sortedY.vec("label") == null ? sortedY.vec(0) : sortedY.vec("label");
    final Vec yValue = sortedY.vec(0);
    Scope.track(xLabel);
    Scope.track(yLabel);

    double lastX = Double.NaN;
    double lastY = Double.NaN;
    long skippedX = 0;
    long skippedY = 0;
    for (int i = 0; i < orderX.length(); i++) {
      if (lastX == xValue.at(i)) {
        skippedX++;
      } else {
        skippedX = 0;
      }
      lastX = xValue.at(i);
      orderX.set(xLabel.at8(i) - 1, i - skippedX);

      if (lastY == yValue.at(i)) {
        skippedY++;
      } else {
        skippedY = 0;
      }
      lastY = yValue.at(i);
      orderY.set(yLabel.at8(i) - 1, i - skippedY);
    }

    final SpearmanCorrelationCoefficientTask spearman = new SpearmanCorrelationCoefficientTask(orderX.mean(), orderY.mean())
            .doAll(orderX, orderY);
    Scope.exit();
    return new ValNum(spearman.getSpearmanCorrelationCoefficient());
  }<commit_msg>Use readers and writers to do ranking for Spearman.<commit_after>package water.rapids.ast.prims.advmath;

import water.DKV;
import water.MRTask;
import water.Scope;
import water.Value;
import water.fvec.Chunk;
import water.fvec.Frame;
import water.fvec.Vec;
import water.rapids.Env;
import water.rapids.Val;
import water.rapids.ast.AstPrimitive;
import water.rapids.ast.AstRoot;
import water.rapids.vals.ValNum;
import water.util.FrameUtils;

public class AstSpearman extends AstPrimitive<AstSpearman> {
  @Override
  public int nargs() {
    return 1 + 3; // Frame ID and ID of two numerical vectors to calculate SCC on.
  }

  @Override
  public String[] args() {
    return new String[]{"frame", "first_column", "second_column"};
  }

  @Override
  public Val apply(Env env, Env.StackHelp stk, AstRoot[] asts) {
    final Value value = DKV.get(asts[1].str());
    if (!value.isFrame()) {
      throw new IllegalArgumentException(String.format("The given key '%s' is not a frame.", asts[1]));
    }

    final Frame originalUnsortedFrame = value.get(Frame.class);
    final int vecIdX = originalUnsortedFrame.find(asts[2].exec(env).getStr());
    final int vecIdY = originalUnsortedFrame.find(asts[3].exec(env).getStr());

    Scope.enter();

    Frame sortedX = new Frame(originalUnsortedFrame.vec(vecIdX).makeCopy());
    Scope.track(sortedX);
    Frame sortedY = new Frame(originalUnsortedFrame.vec(vecIdY).makeCopy());
    Scope.track(sortedY);

    if (!sortedX.vec(0).isCategorical()) {
      FrameUtils.labelRows(sortedX, "label");
      sortedX = sortedX.sort(new int[]{0});
      Scope.track(sortedX);

    }

    if (!sortedY.vec(0).isCategorical()) {
      FrameUtils.labelRows(sortedY, "label");
      sortedY = sortedY.sort(new int[]{0});
      Scope.track(sortedY);
    }

    assert sortedX.numRows() == sortedY.numRows();
    final Vec orderX = Vec.makeZero(sortedX.numRows());
    final Vec orderY = Vec.makeZero(sortedY.numRows());

    final Vec xLabel = sortedX.vec("label") == null ? sortedX.vec(0) : sortedX.vec("label");
    final Vec xValue = sortedX.vec(0);
    final Vec yLabel = sortedY.vec("label") == null ? sortedY.vec(0) : sortedY.vec("label");
    final Vec yValue = sortedY.vec(0);
    Scope.track(xLabel);
    Scope.track(yLabel);


    final Vec.Writer orderXWriter = orderX.open();
    final Vec.Writer orderYWriter = orderY.open();
    final Vec.Reader xValueReader = xValue.new Reader();
    final Vec.Reader yValueReader = yValue.new Reader();
    final Vec.Reader xLabelReader = xLabel.new Reader();
    final Vec.Reader yLabelReader = yLabel.new Reader();
    
    double lastX = Double.NaN;
    double lastY = Double.NaN;
    long skippedX = 0;
    long skippedY = 0;
    for (int i = 0; i < orderX.length(); i++) {
      if (lastX == xValueReader.at(i)) {
        skippedX++;
      } else {
        skippedX = 0;
      }
      lastX = xValueReader.at(i);
      orderXWriter.set(xLabelReader.at8(i) - 1, i - skippedX);

      if (lastY == yValueReader.at(i)) {
        skippedY++;
      } else {
        skippedY = 0;
      }
      lastY = yValueReader.at(i);
      orderYWriter.set(yLabelReader.at8(i) - 1, i - skippedY);
    }
    orderXWriter.close();
    orderYWriter.close();

    final SpearmanCorrelationCoefficientTask spearman = new SpearmanCorrelationCoefficientTask(orderX.mean(), orderY.mean())
            .doAll(orderX, orderY);
    Scope.exit();
    return new ValNum(spearman.getSpearmanCorrelationCoefficient());
  }
{'fraction_non_alphanumeric': 0.14243827160493827, 'fraction_numerical': 0.0070987654320987656, 'mean_word_length': 3.3380187416331997, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5712887', 'n_tokens_mistral': 2272, 'n_tokens_neox': 2159, 'n_words': 474}