<commit_before>package water.api;

import water.MRTask2;
import water.Request2;
import water.UKV;
import water.fvec.Chunk;
import water.fvec.Frame;
import water.fvec.Vec;

import java.util.HashSet;

import static java.util.Arrays.sort;

public class AUC extends Request2 {
  static final int API_WEAVER = 1; // This file has auto-gen'd doc & json fields
  static public DocGen.FieldDoc[] DOC_FIELDS; // Initialized from Auto-Gen code.

  @API(help = "", required = true, filter = Default.class, json=true)
  public Frame actual;

  @API(help="Column of the actual results (will display vertically)", required=true, filter=actualVecSelect.class, json=true)
  public Vec vactual;
  class actualVecSelect extends VecClassSelect { actualVecSelect() { super("actual"); } }

  @API(help = "", required = true, filter = Default.class, json=true)
  public Frame predict;

  @API(help="Column of the predicted results (will display horizontally)", required=true, filter=predictVecSelect.class, json=true)
  public Vec vpredict;
  class predictVecSelect extends VecClassSelect { predictVecSelect() { super("predict"); } }

  @API(help = "Thresholds (optional, e.g. 0:1:0.01 or 0.0,0.2,0.4,0.6,0.8,1.0).", required = false, filter = Default.class, json = true)
  public float[] thresholds;

  @API(help="domain of the actual response")
  private String [] actual_domain;
  @API(help="AUC")
  public double auc;
  @API(help="F1")
  public double f1;
  @API(help="Threshold for max. F1")
  private float best_thresholdF1;


  public double AUC() { return auc; }
  public double err() { return _cms[idx_bestF1].err(); }
  public double F1() { return f1; }
  public double Gini() { return 2*auc-1; }
  public int best_idxF1() { return idx_bestF1; }
  public float best_thresholdF1() { return best_thresholdF1; }

  /* Helpers */
  private int idx_bestF1;
  private double[] _tprs;
  private double[] _fprs;
  private hex.ConfusionMatrix[] _cms;

  @Override public Response serve() {
    Vec va = null, vp;
    // Input handling
    if( vactual==null || vpredict==null )
      throw new IllegalArgumentException("Missing actual or predict!");
    if (vactual.length() != vpredict.length())
      throw new IllegalArgumentException("Both arguments must have the same length!");
    if (!vactual.isInt())
      throw new IllegalArgumentException("Actual column must be integer class labels!");
    if (vpredict.isInt())
      throw new IllegalArgumentException("Predicted column must be a floating point probability!");

    try {
      va = vactual.toEnum(); // always returns TransfVec
      actual_domain = va._domain;
      vp = vpredict;
      // The vectors are from different groups => align them, but properly delete it after computation
      if (!va.group().equals(vp.group())) {
        vp = va.align(vp);
      }

      // make thresholds, if not user-given
      if (thresholds == null) {
        HashSet hs = new HashSet();
        final int bins = (int)Math.min(vpredict.length(), 200l);
        final long stride = Math.max(vpredict.length() / bins, 1);
        for( int i=0; i<bins; ++i) hs.add(new Float(vpredict.at(i*stride))); //data-driven thresholds TODO: use percentiles (from Summary2?)
        for (int i=0;i<51;++i) hs.add(new Float(i/50.)); //always add 0.02-spaced thresholds from 0 to 1

        // created sorted vector of unique thresholds
        thresholds = new float[hs.size()];
        int i=0;
        for (Object h : hs) {thresholds[i++] = (Float)h; }
        sort(thresholds);
      }

      // compute AUC, CMs, and best threshold
      AUCTask at = new AUCTask(thresholds).doAll(va,vp);
      _cms = at.getCMs();
      idx_bestF1 = at.getBestIdxF1();
      _tprs = at.getTPRs();
      _fprs = at.getFPRs();
      auc = at.getAUC();
      f1 = _cms[idx_bestF1].precisionAndRecall();
      best_thresholdF1 = at.getBestThresholdF1();
      return Response.done(this);
    } catch( Throwable t ) {
      return Response.error(t);
    } finally {       // Delete adaptation vectors
      if (va!=null) UKV.remove(va._key);
    }
  }

  @Override public boolean toHTML( StringBuilder sb ) {
    sb.append("<div>");
    DocGen.HTML.section(sb, "Scoring for Binary Classification");
    DocGen.HTML.arrayHead(sb);
//    DocGen.HTML.section(sb, "Predicting: " + actual.names()[actual.find(vactual)]);
    sb.append("<th>AUC</th><th>Gini</th><th>F1</th><th>Threshold for max. F1</th>");
    sb.append("<tr class='warning'>");
    sb.append("<td>"
            + String.format("%5f", AUC()) + "</td><td>"
            + String.format("%5f", Gini()) + "</td><td>"
            + String.format("%5f", F1()) + "</td><td>"
            + String.format("%g", best_thresholdF1()) + "</td>"
    );
    sb.append("</tr>");
    DocGen.HTML.arrayTail(sb);
    _cms[idx_bestF1].toHTML(sb, actual_domain);
    plotROC(sb);
    _cms[idx_bestF1].toHTMLbasic(sb, actual_domain);
    sb.append("\n<script type=\"text/javascript\">");//</script>");
    sb.append("var cms = [\n");
    for(hex.ConfusionMatrix cm:_cms){
      sb.append("\t[\n");
      for(long [] line:cm._arr) {
        sb.append("\t\t[");
        for(long l:line) sb.append(l + ",");
        sb.append("],\n");
      }
      sb.append("\t],\n");
    }
    sb.append("];\n");
    sb.append("function show_cm(i){\n");
    sb.append("\t" + "document.getElementById('TN').innerHTML = cms[i][0][0];\n");
    sb.append("\t" + "document.getElementById('TP').innerHTML = cms[i][1][1];\n");
    sb.append("\t" + "document.getElementById('FN').innerHTML = cms[i][0][1];\n");
    sb.append("\t" + "document.getElementById('FP').innerHTML = cms[i][1][0];\n");
    sb.append("}\n");
    sb.append("</script>\n");
    sb.append("\n<div><b>Confusion Matrix at decision threshold:</b></div><select id=\"select\" onchange='show_cm(this.value)'>\n");
    for(int i = 0; i < _cms.length; ++i)
      sb.append("\t<option value='" + i + "'" + (thresholds[i] == best_thresholdF1()?"selected='selected'":"") +">" + thresholds[i] + "</option>\n");
    sb.append("</select>\n");
    sb.append("</div>");
    return true;
  }

  public double toASCII( StringBuilder sb ) {
    sb.append("AUC: " + String.format("%5f", AUC()));
    sb.append(", Gini: " + String.format("%5f", Gini()));
    sb.append(", F1: " + String.format("%5f", F1()));
    sb.append(", Best threshold for F1: " + String.format("%g", best_thresholdF1()));
    return AUC();
  }

  void plotROC(StringBuilder sb) {
    sb.append("<script type=\"text/javascript\" src='/h2o/js/d3.v3.min.js'></script>");
    sb.append("<div id=\"ROC\">");
    sb.append("<style type=\"text/css\">");
    sb.append(".axis path," +
            ".axis line {\n" +
            "fill: none;\n" +
            "stroke: black;\n" +
            "shape-rendering: crispEdges;\n" +
            "}\n" +

            ".axis text {\n" +
            "font-family: sans-serif;\n" +
            "font-size: 11px;\n" +
            "}\n");

    sb.append("</style>");
    sb.append("<div id=\"rocCurve\" style=\"display:inline;\">");
    sb.append("<script type=\"text/javascript\">");

    sb.append("//Width and height\n");
    sb.append("var w = 500;\n"+<commit_msg>Show CM in AUC ASCII printout.<commit_after>package water.api;

import water.MRTask2;
import water.Request2;
import water.UKV;
import water.fvec.Chunk;
import water.fvec.Frame;
import water.fvec.Vec;

import java.util.HashSet;

import static java.util.Arrays.sort;

public class AUC extends Request2 {
  static final int API_WEAVER = 1; // This file has auto-gen'd doc & json fields
  static public DocGen.FieldDoc[] DOC_FIELDS; // Initialized from Auto-Gen code.

  @API(help = "", required = true, filter = Default.class, json=true)
  public Frame actual;

  @API(help="Column of the actual results (will display vertically)", required=true, filter=actualVecSelect.class, json=true)
  public Vec vactual;
  class actualVecSelect extends VecClassSelect { actualVecSelect() { super("actual"); } }

  @API(help = "", required = true, filter = Default.class, json=true)
  public Frame predict;

  @API(help="Column of the predicted results (will display horizontally)", required=true, filter=predictVecSelect.class, json=true)
  public Vec vpredict;
  class predictVecSelect extends VecClassSelect { predictVecSelect() { super("predict"); } }

  @API(help = "Thresholds (optional, e.g. 0:1:0.01 or 0.0,0.2,0.4,0.6,0.8,1.0).", required = false, filter = Default.class, json = true)
  public float[] thresholds;

  @API(help="domain of the actual response")
  private String [] actual_domain;
  @API(help="AUC")
  public double auc;
  @API(help="F1")
  public double f1;
  @API(help="Threshold for max. F1")
  private float best_thresholdF1;


  public double AUC() { return auc; }
  public double err() { return _cms[idx_bestF1].err(); }
  public double F1() { return f1; }
  public double Gini() { return 2*auc-1; }
  public int best_idxF1() { return idx_bestF1; }
  public float best_thresholdF1() { return best_thresholdF1; }

  /* Helpers */
  private int idx_bestF1;
  private double[] _tprs;
  private double[] _fprs;
  private hex.ConfusionMatrix[] _cms;

  @Override public Response serve() {
    Vec va = null, vp;
    // Input handling
    if( vactual==null || vpredict==null )
      throw new IllegalArgumentException("Missing actual or predict!");
    if (vactual.length() != vpredict.length())
      throw new IllegalArgumentException("Both arguments must have the same length!");
    if (!vactual.isInt())
      throw new IllegalArgumentException("Actual column must be integer class labels!");
    if (vpredict.isInt())
      throw new IllegalArgumentException("Predicted column must be a floating point probability!");

    try {
      va = vactual.toEnum(); // always returns TransfVec
      actual_domain = va._domain;
      vp = vpredict;
      // The vectors are from different groups => align them, but properly delete it after computation
      if (!va.group().equals(vp.group())) {
        vp = va.align(vp);
      }

      // make thresholds, if not user-given
      if (thresholds == null) {
        HashSet hs = new HashSet();
        final int bins = (int)Math.min(vpredict.length(), 200l);
        final long stride = Math.max(vpredict.length() / bins, 1);
        for( int i=0; i<bins; ++i) hs.add(new Float(vpredict.at(i*stride))); //data-driven thresholds TODO: use percentiles (from Summary2?)
        for (int i=0;i<51;++i) hs.add(new Float(i/50.)); //always add 0.02-spaced thresholds from 0 to 1

        // created sorted vector of unique thresholds
        thresholds = new float[hs.size()];
        int i=0;
        for (Object h : hs) {thresholds[i++] = (Float)h; }
        sort(thresholds);
      }

      // compute AUC, CMs, and best threshold
      AUCTask at = new AUCTask(thresholds).doAll(va,vp);
      _cms = at.getCMs();
      idx_bestF1 = at.getBestIdxF1();
      _tprs = at.getTPRs();
      _fprs = at.getFPRs();
      auc = at.getAUC();
      f1 = _cms[idx_bestF1].precisionAndRecall();
      best_thresholdF1 = at.getBestThresholdF1();
      return Response.done(this);
    } catch( Throwable t ) {
      return Response.error(t);
    } finally {       // Delete adaptation vectors
      if (va!=null) UKV.remove(va._key);
    }
  }

  @Override public boolean toHTML( StringBuilder sb ) {
    sb.append("<div>");
    DocGen.HTML.section(sb, "Scoring for Binary Classification");
    DocGen.HTML.arrayHead(sb);
//    DocGen.HTML.section(sb, "Predicting: " + actual.names()[actual.find(vactual)]);
    sb.append("<th>AUC</th><th>Gini</th><th>F1</th><th>Threshold for max. F1</th>");
    sb.append("<tr class='warning'>");
    sb.append("<td>"
            + String.format("%5f", AUC()) + "</td><td>"
            + String.format("%5f", Gini()) + "</td><td>"
            + String.format("%5f", F1()) + "</td><td>"
            + String.format("%g", best_thresholdF1()) + "</td>"
    );
    sb.append("</tr>");
    DocGen.HTML.arrayTail(sb);
    _cms[idx_bestF1].toHTML(sb, actual_domain);
    plotROC(sb);
    _cms[idx_bestF1].toHTMLbasic(sb, actual_domain);
    sb.append("\n<script type=\"text/javascript\">");//</script>");
    sb.append("var cms = [\n");
    for(hex.ConfusionMatrix cm:_cms){
      sb.append("\t[\n");
      for(long [] line:cm._arr) {
        sb.append("\t\t[");
        for(long l:line) sb.append(l + ",");
        sb.append("],\n");
      }
      sb.append("\t],\n");
    }
    sb.append("];\n");
    sb.append("function show_cm(i){\n");
    sb.append("\t" + "document.getElementById('TN').innerHTML = cms[i][0][0];\n");
    sb.append("\t" + "document.getElementById('TP').innerHTML = cms[i][1][1];\n");
    sb.append("\t" + "document.getElementById('FN').innerHTML = cms[i][0][1];\n");
    sb.append("\t" + "document.getElementById('FP').innerHTML = cms[i][1][0];\n");
    sb.append("}\n");
    sb.append("</script>\n");
    sb.append("\n<div><b>Confusion Matrix at decision threshold:</b></div><select id=\"select\" onchange='show_cm(this.value)'>\n");
    for(int i = 0; i < _cms.length; ++i)
      sb.append("\t<option value='" + i + "'" + (thresholds[i] == best_thresholdF1()?"selected='selected'":"") +">" + thresholds[i] + "</option>\n");
    sb.append("</select>\n");
    sb.append("</div>");
    return true;
  }

  public double toASCII( StringBuilder sb ) {
    sb.append(_cms[idx_bestF1].toString());
    sb.append("AUC: " + String.format("%5f", AUC()));
    sb.append(", Gini: " + String.format("%5f", Gini()));
    sb.append(", F1: " + String.format("%5f", F1()));
    sb.append(", Best threshold for F1: " + String.format("%g", best_thresholdF1()));
    return AUC();
  }

  void plotROC(StringBuilder sb) {
    sb.append("<script type=\"text/javascript\" src='/h2o/js/d3.v3.min.js'></script>");
    sb.append("<div id=\"ROC\">");
    sb.append("<style type=\"text/css\">");
    sb.append(".axis path," +
            ".axis line {\n" +
            "fill: none;\n" +
            "stroke: black;\n" +
            "shape-rendering: crispEdges;\n" +
            "}\n" +

            ".axis text {\n" +
            "font-family: sans-serif;\n" +
            "font-size: 11px;\n" +
            "}\n");

    sb.append("</style>");
    sb.append("<div id=\"rocCurve\" style=\"display:inline;\">");
    sb.append("<script type=\"text/javascript\">");

    sb.append("//Width and height\n");
    sb.append("var w = 500;\n"+
{'fraction_non_alphanumeric': 0.1843555927709162, 'fraction_numerical': 0.012909078222036146, 'mean_word_length': 3.146990740740741, 'pattern_counts': {'":': 2, '<': 85, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5374139', 'n_tokens_mistral': 4881, 'n_tokens_neox': 4698, 'n_words': 1081}