<commit_before>package zephyr.plugin.core.internal.observations;

/*
 * From Mike Sokolsky code 
 */
public class ObsStat {
  public final String label;
  final private double rangeMin;
  final private double rangeMax;
  private final int observationIndex;
  private int numSamples;
  double mean;
  double decayMax;
  double decayMin;
  double max;
  double min;
  private double stdCalc;
  private double stdDev;
  double current;
  double length;
  private double barMax;
  double barMin;

  public ObsStat(String label, int observationIndex, double rangeMin, double rangeMax) {
    this.observationIndex = observationIndex;
    this.rangeMin = rangeMin;
    this.rangeMax = rangeMax;
    this.label = label;
  }

  public void updateValue(double[] currentObservation) {
    double newVal = currentObservation[observationIndex];

    stdCalc += numSamples * Math.pow(newVal - mean, 2) / (numSamples + 1);
    numSamples++;
    mean += (newVal - mean) / numSamples;
    stdDev = Math.sqrt(stdCalc / (numSamples + 1));

    decayMax -= (max - min) / 10000;
    decayMin += (max - min) / 10000;

    decayMin = Math.min(decayMin, newVal);
    decayMax = Math.max(decayMax, newVal);

    min = Math.min(min, newVal);
    max = Math.max(max, newVal);
    current = newVal;
    barMax = Math.max(max, rangeMax);
    barMin = Math.min(min, rangeMin);
    length = barMax - barMin;
  }

  public double scaledStdDev() {
    return stdDev / length;
  }
}
<commit_msg>Fix a bug in min/max initialisation<commit_after>package zephyr.plugin.core.internal.observations;

/*
 * From Mike Sokolsky code 
 */
public class ObsStat {
  public final String label;
  final private double rangeMin;
  final private double rangeMax;
  private final int observationIndex;
  private int numSamples;
  double mean;
  double decayMax;
  double decayMin;
  double max = -Double.MAX_VALUE;
  double min = Double.MAX_VALUE;
  private double stdCalc;
  private double stdDev;
  double current;
  double length;
  private double barMax;
  double barMin;

  public ObsStat(String label, int observationIndex, double rangeMin, double rangeMax) {
    this.observationIndex = observationIndex;
    this.rangeMin = rangeMin;
    this.rangeMax = rangeMax;
    this.label = label;
  }

  public void updateValue(double[] currentObservation) {
    double newVal = currentObservation[observationIndex];

    stdCalc += numSamples * Math.pow(newVal - mean, 2) / (numSamples + 1);
    numSamples++;
    mean += (newVal - mean) / numSamples;
    stdDev = Math.sqrt(stdCalc / (numSamples + 1));

    decayMax -= (max - min) / 10000;
    decayMin += (max - min) / 10000;

    decayMin = Math.min(decayMin, newVal);
    decayMax = Math.max(decayMax, newVal);

    min = Math.min(min, newVal);
    max = Math.max(max, newVal);
    current = newVal;
    barMax = Math.max(max, rangeMax);
    barMin = Math.min(min, rangeMin);
    length = barMax - barMin;
  }

  public double scaledStdDev() {
    return stdDev / length;
  }
}

{'fraction_non_alphanumeric': 0.10459269192088501, 'fraction_numerical': 0.008716057660073752, 'mean_word_length': 3.3246376811594205, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3230996', 'n_tokens_mistral': 996, 'n_tokens_neox': 943, 'n_words': 267}