<commit_before>#include "strm.h"
#include <math.h>

struct sum_data {
  double sum;
  double c;
  strm_int num;
  strm_value func;
};

static int
iter_sum(strm_stream* strm, strm_value data)
{
  struct sum_data* d = strm->data;
  double x, t;

  if (!strm_number_p(data)) {
    return STRM_NG;
  }
  x = strm_value_float(data);
  t = d->sum + x;
  if (fabs(d->sum) >= fabs(x))
    d->c += ((d->sum - t) + x);
  else
    d->c += ((x - t) + d->sum);
  d->sum = t;
  d->num++;
  return STRM_OK;
}

static strm_value
convert_number(strm_stream* strm, strm_value data, strm_value func)
{
  strm_value val;

  if (strm_funcall(strm, func, 1, &data, &val) == STRM_NG) {
    return STRM_NG;
  }
  if (!strm_number_p(val)) {
    strm_raise(strm, "number required");
    return STRM_NG;
  }
  return val;
}

static int
iter_sumf(strm_stream* strm, strm_value data)
{
  struct sum_data* d = strm->data;
  double x, t;

  data = convert_number(strm, data, d->func);
  x = strm_value_float(data);
  t = d->sum + x;
  if (fabs(d->sum) >= fabs(x))
    d->c += ((d->sum - t) + x);
  else
    d->c += ((x - t) + d->sum);
  d->sum = t;
  d->num++;
  return STRM_OK;
}

static int
sum_finish(strm_stream* strm, strm_value data)
{
  struct sum_data* d = strm->data;

  strm_emit(strm, strm_float_value(d->sum+d->c), NULL);
  return STRM_OK;
}

static int
avg_finish(strm_stream* strm, strm_value data)
{
  struct sum_data* d = strm->data;

  strm_emit(strm, strm_float_value((d->sum+d->c)/d->num), NULL);
  return STRM_OK;
}

static int
exec_sum_avg(strm_stream* strm, int argc, strm_value* args, strm_value* ret, int avg)
{
  struct sum_data* d;
  strm_value func;

  strm_get_args(strm, argc, args, "|v", &func);
  d = malloc(sizeof(struct sum_data));
  if (!d) return STRM_NG;
  d->sum = 0;
  d->c = 0;
  d->num = 0;
  if (argc == 0) {
    d->func = strm_nil_value();
    *ret = strm_stream_value(strm_stream_new(strm_filter, iter_sum,
                                             avg ? avg_finish : sum_finish, (void*)d));
  }
  else {
    d->func = func;
    *ret = strm_stream_value(strm_stream_new(strm_filter, iter_sumf,
                                             avg ? avg_finish : sum_finish, (void*)d));
  }
  return STRM_OK;
}

static int
exec_sum(strm_stream* strm, int argc, strm_value* args, strm_value* ret)
{
  return exec_sum_avg(strm, argc, args, ret, FALSE);
}

static int
exec_avg(strm_stream* strm, int argc, strm_value* args, strm_value* ret)
{
  return exec_sum_avg(strm, argc, args, ret, TRUE);
}

static int
ary_sum_avg(strm_stream* strm, int argc, strm_value* args, strm_value* ret, int avg)
{
  int i, len;
  strm_value* v;
  double sum = 0;
  double c = 0;
  strm_value func;

  strm_get_args(strm, argc, args, "a|v", &v, &len, &func);
  if (argc == 0) {
    for (i=0; i<len; i++) {
      double y = strm_value_float(v[i]) - c;
      double t = sum + y;
      c = (t - sum) - y;
      sum =  t;
    }
  }
  else {
    for (i=0; i<len; i++) {
      strm_value val;
      double f, y, t;

      val = convert_number(strm, v[i], func);
      f = strm_value_float(val);
      y = f - c;
      t = sum + y;
      c = (t - sum) - y;
      sum =  t;
    }
  }
  if (avg) {
    *ret = strm_float_value(sum/len);
  }
  else {
    *ret = strm_float_value(sum);
  }
  return STRM_OK;
}

static int
ary_sum(strm_stream* strm, int argc, strm_value* args, strm_value* ret)
{
  return ary_sum_avg(strm, argc, args, ret, FALSE);
}

static int
ary_avg(strm_stream* strm, int argc, strm_value* args, strm_value* ret)
{
  return ary_sum_avg(strm, argc, args, ret, TRUE);
}

enum stdev_mode {
  mode_stdev,
  mode_variance,
  mode_mean_stdev,
  mode_mean_variance,
};<commit_msg>Apply Kahan-Babushka to sum(array) as well.<commit_after>#include "strm.h"
#include <math.h>

struct sum_data {
  double sum;
  double c;
  strm_int num;
  strm_value func;
};

static int
iter_sum(strm_stream* strm, strm_value data)
{
  struct sum_data* d = strm->data;
  double x, t;

  if (!strm_number_p(data)) {
    return STRM_NG;
  }
  x = strm_value_float(data);
  t = d->sum + x;
  if (fabs(d->sum) >= fabs(x))
    d->c += ((d->sum - t) + x);
  else
    d->c += ((x - t) + d->sum);
  d->sum = t;
  d->num++;
  return STRM_OK;
}

static strm_value
convert_number(strm_stream* strm, strm_value data, strm_value func)
{
  strm_value val;

  if (strm_funcall(strm, func, 1, &data, &val) == STRM_NG) {
    return STRM_NG;
  }
  if (!strm_number_p(val)) {
    strm_raise(strm, "number required");
    return STRM_NG;
  }
  return val;
}

static int
iter_sumf(strm_stream* strm, strm_value data)
{
  struct sum_data* d = strm->data;
  double x, t;

  data = convert_number(strm, data, d->func);
  x = strm_value_float(data);
  t = d->sum + x;
  if (fabs(d->sum) >= fabs(x))
    d->c += ((d->sum - t) + x);
  else
    d->c += ((x - t) + d->sum);
  d->sum = t;
  d->num++;
  return STRM_OK;
}

static int
sum_finish(strm_stream* strm, strm_value data)
{
  struct sum_data* d = strm->data;

  strm_emit(strm, strm_float_value(d->sum+d->c), NULL);
  return STRM_OK;
}

static int
avg_finish(strm_stream* strm, strm_value data)
{
  struct sum_data* d = strm->data;

  strm_emit(strm, strm_float_value((d->sum+d->c)/d->num), NULL);
  return STRM_OK;
}

static int
exec_sum_avg(strm_stream* strm, int argc, strm_value* args, strm_value* ret, int avg)
{
  struct sum_data* d;
  strm_value func;

  strm_get_args(strm, argc, args, "|v", &func);
  d = malloc(sizeof(struct sum_data));
  if (!d) return STRM_NG;
  d->sum = 0;
  d->c = 0;
  d->num = 0;
  if (argc == 0) {
    d->func = strm_nil_value();
    *ret = strm_stream_value(strm_stream_new(strm_filter, iter_sum,
                                             avg ? avg_finish : sum_finish, (void*)d));
  }
  else {
    d->func = func;
    *ret = strm_stream_value(strm_stream_new(strm_filter, iter_sumf,
                                             avg ? avg_finish : sum_finish, (void*)d));
  }
  return STRM_OK;
}

static int
exec_sum(strm_stream* strm, int argc, strm_value* args, strm_value* ret)
{
  return exec_sum_avg(strm, argc, args, ret, FALSE);
}

static int
exec_avg(strm_stream* strm, int argc, strm_value* args, strm_value* ret)
{
  return exec_sum_avg(strm, argc, args, ret, TRUE);
}

static int
ary_sum_avg(strm_stream* strm, int argc, strm_value* args, strm_value* ret, int avg)
{
  int i, len;
  strm_value* v;
  double sum = 0;
  double c = 0;
  strm_value func;

  strm_get_args(strm, argc, args, "a|v", &v, &len, &func);
  if (argc == 1) {
    for (i=0; i<len; i++) {
      double x = strm_value_float(v[i]);
      double t = sum + x;
      if (fabs(sum) >= fabs(x))
        c += ((sum - t) + x);
      else
        c += ((x - t) + sum);
      sum =  t;
    }
  }
  else {
    for (i=0; i<len; i++) {
      strm_value val;
      double x, t;

      val = convert_number(strm, v[i], func);
      x = strm_value_float(val);
      t = sum + x;
      if (fabs(sum) >= fabs(x))
        c += ((sum - t) + x);
      else
        c += ((x - t) + sum);
      sum =  t;
    }
  }
  sum += c;
  if (avg) {
    *ret = strm_float_value(sum/len);
  }
  else {
    *ret = strm_float_value(sum);
  }
  return STRM_OK;
}

static int
ary_sum(strm_stream* strm, int argc, strm_value* args, strm_value* ret)
{
  return ary_sum_avg(strm, argc, args, ret, FALSE);
}

static int
ary_avg(strm_stream* strm, int argc, strm_value* args, strm_value* ret)
{
  return ary_sum_avg(strm, argc, args, ret, TRUE);
}

enum stdev_mode {
  mode_stdev,
  mode_variance,
  mode_mean_stdev,
  mode_mean_variance,
};
{'fraction_non_alphanumeric': 0.15491083311152515, 'fraction_numerical': 0.002661698163428267, 'mean_word_length': 2.90795631825273, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1812279', 'n_tokens_mistral': 3258, 'n_tokens_neox': 3226, 'n_words': 758}