<commit_before>/*
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0.  If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 * Copyright 2008-2015 MonetDB B.V.
 */

#include "monetdb_config.h"
#include "mal.h"
#include "mal_exception.h"

#ifdef WIN32
#define aggr_export extern __declspec(dllexport)
#else
#define aggr_export extern
#endif

/*
 * grouped aggregates
 */
static str
AGGRgrouped(bat *retval1, bat *retval2, BAT *b, BAT *g, BAT *e, int tp,
			BAT *(*grpfunc1)(BAT *, BAT *, BAT *, BAT *, int, int, int),
			gdk_return (*grpfunc2)(BAT **, BAT **, BAT *, BAT *, BAT *, BAT *, int, int, int),
			BAT *(*quantilefunc)(BAT *, BAT *, BAT *, BAT *, int, double, int, int),
			BAT *quantile,
			int skip_nils,
			const char *malfunc)
{
	BAT *bn, *cnts = NULL;
	double qvalue;

   /* one of grpfunc1, grpfunc2 and quantilefunc is non-NULL and the others are */
	assert((grpfunc1 != NULL && grpfunc2 == NULL && quantilefunc == NULL) ||
			(grpfunc1 == NULL && grpfunc2 != NULL && quantilefunc == NULL) ||
			(grpfunc1 == NULL && grpfunc2 == NULL && quantilefunc != NULL) );
	/* if retval2 is non-NULL, we must have grpfunc2 */
	assert(retval2 == NULL || grpfunc2 != NULL);
	assert(quantile == NULL || quantilefunc != NULL);

	if (b == NULL || g == NULL || e == NULL) {
		if (b)
			BBPunfix(b->batCacheid);
		if (g)
			BBPunfix(g->batCacheid);
		if (e)
			BBPunfix(e->batCacheid);
		throw(MAL, malfunc, RUNTIME_OBJECT_MISSING);
	}
	assert(BAThdense(b));
	assert(BAThdense(g));
	assert(BAThdense(e));
	assert(b->hseqbase == g->hseqbase);
	assert(BATcount(b) == BATcount(g));
	if (tp == TYPE_any && (grpfunc1 == BATgroupmedian || quantilefunc == BATgroupquantile))
		tp = b->ttype;
	if (grpfunc1)
		bn = (*grpfunc1)(b, g, e, NULL, tp, skip_nils, 1);
	if (quantilefunc) {
		assert(BATcount(quantile)>0);
		assert(quantile->ttype == TYPE_dbl);
		qvalue = ((const double *)Tloc(quantile, BUNfirst(quantile)))[0];
		if (qvalue <  0|| qvalue > 1) {
			char *s;
			s = createException(MAL, malfunc, "quantile value of %f is not in range [0,1]", qvalue);
			return s;
		}
		bn = (*quantilefunc)(b, g, e, NULL, tp, qvalue, skip_nils, 1);
	}
	if (grpfunc2 && (*grpfunc2)(&bn, retval2 ? &cnts : NULL, b, g, e, NULL, tp, skip_nils, 1) != GDK_SUCCEED)
		bn = NULL;
	if (bn != NULL && (grpfunc1 == BATgroupmin || grpfunc1 == BATgroupmax)) {
		BAT *t = BATproject(bn, b);
		BBPunfix(bn->batCacheid);
		bn = t;
	}
	BBPunfix(b->batCacheid);
	BBPunfix(g->batCacheid);
	BBPunfix(e->batCacheid);
	if (bn == NULL) {
		char *errbuf = GDKerrbuf;
		char *s;

		if (errbuf && *errbuf) {
			if (strncmp(errbuf, "!ERROR: ", 8) == 0)
				errbuf += 8;
			if (strchr(errbuf, '!') == errbuf + 5) {
				s = createException(MAL, malfunc, "%s", errbuf);
			} else if ((s = strchr(errbuf, ':')) != NULL && s[1] == ' ') {
				s = createException(MAL, malfunc, "%s", s + 2);
			} else {
				s = createException(MAL, malfunc, "%s", errbuf);
			}
			*GDKerrbuf = 0;
			return s;
		}
		throw(MAL, malfunc, OPERATION_FAILED);
	}
	*retval1 = bn->batCacheid;
	BBPkeepref(bn->batCacheid);
	if (retval2) {
		*retval2 = cnts->batCacheid;
		BBPkeepref(cnts->batCacheid);
	}
	return MAL_SUCCEED;
}

static str
AGGRgrouped3(bat *retval1, bat *retval2, const bat *bid, const bat *gid, const bat *eid, int tp,
			 BAT *(*grpfunc1)(BAT *, BAT *, BAT *, BAT *, int, int, int),
			 gdk_return (*grpfunc2)(BAT **, BAT **, BAT *, BAT *, BAT *, BAT *, int, int, int),
			 int skip_nils,
			 const char *malfunc)
{
	BAT *b, *g, *e;

	b = BATdescriptor(*bid);	/* [head,value] */
	g = BATdescriptor(*gid);	/* [head,gid] */
	e = BATdescriptor(*eid);	/* [gid,any] */
	return AGGRgrouped(retval1, retval2, b, g, e, tp, grpfunc1, grpfunc2, NULL, 0, skip_nils, malfunc);
}

static str
AGGRgrouped2(bat *retval1, bat *retval2, const bat *bid, const bat *eid, int tp,
			 BAT *(*grpfunc1)(BAT *, BAT *, BAT *, BAT *, int, int, int),
			 gdk_return (*grpfunc2)(BAT **, BAT **, BAT *, BAT *, BAT *, BAT *, int, int, int),
			 int skip_nils,
			 const char *malfunc)
{
	BAT *b, *g, *e;

	b = BATdescriptor(*bid);	/* [gid,value] */
	if (b == NULL)
		throw(MAL, "aggr.sum", RUNTIME_OBJECT_MISSING);
	g = BATmirror(BATmark(b, 0)); /* [dense,gid] */
	e = BATmirror(BATmark(BATmirror(b), 0)); /* [dense,value] */
	BBPunfix(b->batCacheid);
	b = e;
	e = BATdescriptor(*eid);	/* [gid,any] */
	return AGGRgrouped(retval1, retval2, b, g, e, tp, grpfunc1, grpfunc2, NULL, 0, skip_nils, malfunc);
}

aggr_export str AGGRsum3_bte(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRsum3_bte(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_bte,
						BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRsum3_sht(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRsum3_sht(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_sht,
						BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRsum3_int(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRsum3_int(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_int,
						BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRsum3_wrd(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRsum3_wrd(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_wrd,
						BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRsum3_lng(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRsum3_lng(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_lng,
						BATgroupsum, NULL, 1, "aggr.sum");
}

#ifdef HAVE_HGE
aggr_export str AGGRsum3_hge(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRsum3_hge(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_hge,
						BATgroupsum, NULL, 1, "aggr.sum");
}
#endif

aggr_export str AGGRsum3_flt(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRsum3_flt(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_flt,
						BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRsum3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRsum3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_dbl,
						BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRsum2_bte(bat *retval, const bat *bid, const bat *eid);
str
AGGRsum2_bte(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_bte, BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRsum2_sht(bat *retval, const bat *bid, const bat *eid);
str
AGGRsum2_sht(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_sht, BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRsum2_int(bat *retval, const bat *bid, const bat *eid);
str
AGGRsum2_int(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_int, BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRsum2_wrd(bat *retval, const bat *bid, const bat *eid);
str
AGGRsum2_wrd(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_wrd, BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRsum2_lng(bat *retval, const bat *bid, const bat *eid);
str
AGGRsum2_lng(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_lng, BATgroupsum, NULL, 1, "aggr.sum");
}

#ifdef HAVE_HGE
aggr_export str AGGRsum2_hge(bat *retval, const bat *bid, const bat *eid);
str
AGGRsum2_hge(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_hge, BATgroupsum, NULL, 1, "aggr.sum");
}
#endif

aggr_export str AGGRsum2_flt(bat *retval, const bat *bid, const bat *eid);
str
AGGRsum2_flt(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_flt, BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRsum2_dbl(bat *retval, const bat *bid, const bat *eid);
str
AGGRsum2_dbl(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_dbl, BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRprod3_bte(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRprod3_bte(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_bte,
						BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRprod3_sht(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRprod3_sht(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_sht,
						BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRprod3_int(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRprod3_int(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_int,
						BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRprod3_wrd(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRprod3_wrd(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_wrd,
						BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRprod3_lng(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRprod3_lng(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_lng,
						BATgroupprod, NULL, 1, "aggr.prod");
}

#ifdef HAVE_HGE
aggr_export str AGGRprod3_hge(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRprod3_hge(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_hge,
						BATgroupprod, NULL, 1, "aggr.prod");
}
#endif

aggr_export str AGGRprod3_flt(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRprod3_flt(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_flt,
						BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRprod3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRprod3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_dbl,
						BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRprod2_bte(bat *retval, const bat *bid, const bat *eid);
str
AGGRprod2_bte(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_bte, BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRprod2_sht(bat *retval, const bat *bid, const bat *eid);
str
AGGRprod2_sht(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_sht, BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRprod2_int(bat *retval, const bat *bid, const bat *eid);
str
AGGRprod2_int(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_int, BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRprod2_wrd(bat *retval, const bat *bid, const bat *eid);
str
AGGRprod2_wrd(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_wrd, BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRprod2_lng(bat *retval, const bat *bid, const bat *eid);
str
AGGRprod2_lng(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_lng, BATgroupprod, NULL, 1, "aggr.prod");
}

#ifdef HAVE_HGE
aggr_export str AGGRprod2_hge(bat *retval, const bat *bid, const bat *eid);
str
AGGRprod2_hge(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_hge, BATgroupprod, NULL, 1, "aggr.prod");
}
#endif

aggr_export str AGGRprod2_flt(bat *retval, const bat *bid, const bat *eid);
str
AGGRprod2_flt(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_flt, BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRprod2_dbl(bat *retval, const bat *bid, const bat *eid);
str
AGGRprod2_dbl(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_dbl, BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRavg13_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRavg13_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_dbl,
						NULL, BATgroupavg, 1, "aggr.avg");
}

aggr_export str AGGRavg12_dbl(bat *retval, const bat *bid, const bat *eid);
str
AGGRavg12_dbl(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_dbl, NULL, BATgroupavg, 1, "aggr.avg");
}

aggr_export str AGGRavg23_dbl(bat *retval1, bat *retval2, const bat *bid, const bat *gid, const bat *eid);
str
AGGRavg23_dbl(bat *retval1, bat *retval2, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval1, retval2, bid, gid, eid, TYPE_dbl,
						NULL, BATgroupavg, 1, "aggr.avg");
}

aggr_export str AGGRavg22_dbl(bat *retval1, bat *retval2, const bat *bid, const bat *eid);
str
AGGRavg22_dbl(bat *retval1, bat *retval2, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval1, retval2, bid, eid, TYPE_dbl, NULL, BATgroupavg, 1, "aggr.avg");
}

aggr_export str AGGRstdev3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRstdev3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_dbl,
						BATgroupstdev_sample, NULL, 1, "aggr.stdev");
}

aggr_export str AGGRstdev2_dbl(bat *retval, const bat *bid, const bat *eid);
str
AGGRstdev2_dbl(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_dbl, BATgroupstdev_sample, NULL, 1, "aggr.stdev");
}

aggr_export str AGGRstdevp3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRstdevp3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_dbl,
						BATgroupstdev_population, NULL, 1, "aggr.stdevp");
}

aggr_export str AGGRstdevp2_dbl(bat *retval, const bat *bid, const bat *eid);
str
AGGRstdevp2_dbl(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_dbl, BATgroupstdev_population, NULL, 1, "aggr.stdevp");
}

aggr_export str AGGRvariance3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRvariance3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_dbl,
						BATgroupvariance_sample, NULL, 1, "aggr.variance");
}

aggr_export str AGGRvariance2_dbl(bat *retval, const bat *bid, const bat *eid);
str
AGGRvariance2_dbl(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_dbl, BATgroupvariance_sample, NULL, 1, "aggr.variance");
}

aggr_export str AGGRvariancep3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRvariancep3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_dbl,
						BATgroupvariance_population, NULL, 1, "aggr.variancep");
}

aggr_export str AGGRvariancep2_dbl(bat *retval, const bat *bid, const bat *eid);
str
AGGRvariancep2_dbl(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_dbl, BATgroupvariance_population, NULL, 1, "aggr.variancep");
}

aggr_export str AGGRcount3(bat *retval, const bat *bid, const bat *gid, const bat *eid, const bit *ignorenils);
str
AGGRcount3(bat *retval, const bat *bid, const bat *gid, const bat *eid, const bit *ignorenils)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_wrd,
						BATgroupcount, NULL, *ignorenils, "aggr.count");
}

aggr_export str AGGRcount3nonils(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRcount3nonils(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_wrd,
						BATgroupcount, NULL, 1, "aggr.count");
}

aggr_export str AGGRcount3nils(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRcount3nils(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_wrd,
						BATgroupcount, NULL, 0, "aggr.count");
}

aggr_export str AGGRcount2(bat *retval, const bat *bid, const bat *eid, const bit *ignorenils);
str
AGGRcount2(bat *retval, const bat *bid, const bat *eid, const bit *ignorenils)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_wrd,
						BATgroupcount, NULL, *ignorenils, "aggr.count");
}

aggr_export str AGGRcount2nonils(bat *retval, const bat *bid, const bat *eid);
str
AGGRcount2nonils(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_wrd,
						BATgroupcount, NULL, 1, "aggr.count");
}

aggr_export str AGGRcount2nils(bat *retval, const bat *bid, const bat *eid);
str
AGGRcount2nils(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_wrd,
						BATgroupcount, NULL, 0, "aggr.count");
}

aggr_export str AGGRsize2nils(bat *retval, const bat *bid, const bat *eid);
str
AGGRsize2nils(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_wrd,
						BATgroupsize, NULL, 0, "aggr.size");
}

aggr_export str AGGRmin3(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRmin3(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_oid,
						BATgroupmin, NULL, 0, "aggr.min");
}

aggr_export str AGGRmin2(bat *retval, const bat *bid, const bat *eid);
str
AGGRmin2(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_oid, BATgroupmin, NULL, 0, "aggr.min");
}

aggr_export str AGGRmax3(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRmax3(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_oid,
						BATgroupmax, NULL, 0, "aggr.max");
}

aggr_export str AGGRmax2(bat *retval, const bat *bid, const bat *eid);
str
AGGRmax2(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_oid, BATgroupmax, NULL, 0, "aggr.max");
}

aggr_export str AGGRmedian3(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRmedian3(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_any,
						BATgroupmedian, NULL, 0, "aggr.median");
}


// XXX: when are these functions called?
aggr_export str AGGRquantile3(bat *retval, const bat *bid, const bat *gid, const bat *eid, const bat *quantile);
str
AGGRquantile3(bat *retval, const bat *bid, const bat *gid, const bat *eid, const bat *quantile)
{
	// this is inlined from AGGRgrouped3 to avoid changing all the other functions for now
	BAT *b, *g, *e, *q;
	b = BATdescriptor(*bid);	/* [head,value] */
	g = BATdescriptor(*gid);	/* [head,gid] */
	e = BATdescriptor(*eid);	/* [gid,any] */
	q = BATdescriptor(*quantile);
	return AGGRgrouped(retval, NULL, b, g, e, TYPE_any, NULL, NULL,
					   BATgroupquantile, q, 0, "aggr.quantile");
}

static str
AGGRsubgroupedExt(bat *retval1, bat *retval2, const bat *bid, const bat *gid, const bat *eid, const bat *sid,
			   int skip_nils, int abort_on_error, int tp,
			   BAT *(*grpfunc1)(BAT *, BAT *, BAT *, BAT *, int, int, int),
			   gdk_return (*grpfunc2)(BAT **, BAT **, BAT *, BAT *, BAT *, BAT *, int, int, int),
			   BAT *(*quantilefunc)(BAT *, BAT *, BAT *, BAT *, int, double, int, int),
			   const bat *quantile,
			   const char *malfunc)
{
	BAT *b, *g, *e, *s, *bn = NULL, *cnts, *q = NULL;
	double qvalue;

   /* one of grpfunc1, grpfunc2 and quantilefunc is non-NULL and the others are */
	assert((grpfunc1 && grpfunc2 == NULL && quantilefunc == NULL) ||
			(grpfunc1 == NULL && grpfunc2 && quantilefunc == NULL) ||
			(grpfunc1 == NULL && grpfunc2 == NULL && quantilefunc) );

	/* if retval2 is non-NULL, we must have grpfunc2 */
	assert(retval2 == NULL || grpfunc2 != NULL);

	b = BATdescriptor(*bid);
	g = gid ? BATdescriptor(*gid) : NULL;
	e = eid ? BATdescriptor(*eid) : NULL;
	q = quantile ? BATdescriptor(*quantile) : NULL;

	if (b == NULL || (gid != NULL && g == NULL) || (eid != NULL && e == NULL)) {
		if (b)
			BBPunfix(b->batCacheid);
		if (g)
			BBPunfix(g->batCacheid);
		if (e)
			BBPunfix(e->batCacheid);
		throw(MAL, malfunc, RUNTIME_OBJECT_MISSING);
	}
	if (tp == TYPE_any && (grpfunc1 == BATgroupmedian || quantilefunc == BATgroupquantile))
		tp = b->ttype;

	if (sid) {
		s = BATdescriptor(*sid);
		if (s == NULL) {
			BBPunfix(b->batCacheid);
			if (g)
				BBPunfix(g->batCacheid);
			if (e)
				BBPunfix(e->batCacheid);
			throw(MAL, malfunc, RUNTIME_OBJECT_MISSING);
		}
	} else {
		if (!BAThdense(b)) {
			/* XXX backward compatibility code: ignore non-dense head, but
			 * only if no candidate list */
			s = BATmirror(BATmark(BATmirror(b), 0));
			BBPunfix(b->batCacheid);
			b = s;
		}
		s = NULL;
	}
	if (grpfunc1)
		bn = (*grpfunc1)(b, g, e, s, tp, skip_nils, abort_on_error);
	if (quantilefunc) {
		assert(BATcount(q) > 0 || BATcount(b) == 0);
		assert(q->ttype == TYPE_dbl);
		if (BATcount(q) == 0) {
			qvalue = 0.5;
		} else {
			qvalue = ((const dbl *)Tloc(q, BUNfirst(q)))[0];
			if (qvalue <  0|| qvalue > 1) {
				char *s;
				s = createException(MAL, malfunc, "quantile value of %f is not in range [0,1]", qvalue);
				return s;
			}
		}
		bn = (*quantilefunc)(b, g, e, s, tp, qvalue, skip_nils, abort_on_error);
	}
	if (grpfunc2 && (*grpfunc2)(&bn, retval2 ? &cnts : NULL, b, g, e, s, tp, skip_nils, abort_on_error) != GDK_SUCCEED)
		bn = NULL;

	BBPunfix(b->batCacheid);
	if (g)
		BBPunfix(g->batCacheid);
	if (e)
		BBPunfix(e->batCacheid);
	if (s)
		BBPunfix(s->batCacheid);
	if (bn == NULL) {
		char *errbuf = GDKerrbuf;
		char *s;

		if (errbuf && *errbuf) {
			if (strncmp(errbuf, "!ERROR: ", 8) == 0)
				errbuf += 8;
			if (strchr(errbuf, '!') == errbuf + 5) {
				s = createException(MAL, malfunc, "%s", errbuf);
			} else if ((s = strchr(errbuf, ':')) != NULL && s[1] == ' ') {
				s = createException(MAL, malfunc, "%s", s + 2);
			} else {
				s = createException(MAL, malfunc, "%s", errbuf);
			}
			*GDKerrbuf = 0;
			return s;
		}
		throw(MAL, malfunc, OPERATION_FAILED);
	}
	*retval1 = bn->batCacheid;<commit_msg>unfix quantile bat fixes Bug 3825<commit_after>/*
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0.  If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 * Copyright 2008-2015 MonetDB B.V.
 */

#include "monetdb_config.h"
#include "mal.h"
#include "mal_exception.h"

#ifdef WIN32
#define aggr_export extern __declspec(dllexport)
#else
#define aggr_export extern
#endif

/*
 * grouped aggregates
 */
static str
AGGRgrouped(bat *retval1, bat *retval2, BAT *b, BAT *g, BAT *e, int tp,
			BAT *(*grpfunc1)(BAT *, BAT *, BAT *, BAT *, int, int, int),
			gdk_return (*grpfunc2)(BAT **, BAT **, BAT *, BAT *, BAT *, BAT *, int, int, int),
			BAT *(*quantilefunc)(BAT *, BAT *, BAT *, BAT *, int, double, int, int),
			BAT *quantile,
			int skip_nils,
			const char *malfunc)
{
	BAT *bn, *cnts = NULL;
	double qvalue;

   /* one of grpfunc1, grpfunc2 and quantilefunc is non-NULL and the others are */
	assert((grpfunc1 != NULL && grpfunc2 == NULL && quantilefunc == NULL) ||
			(grpfunc1 == NULL && grpfunc2 != NULL && quantilefunc == NULL) ||
			(grpfunc1 == NULL && grpfunc2 == NULL && quantilefunc != NULL) );
	/* if retval2 is non-NULL, we must have grpfunc2 */
	assert(retval2 == NULL || grpfunc2 != NULL);
	assert(quantile == NULL || quantilefunc != NULL);

	if (b == NULL || g == NULL || e == NULL) {
		if (b)
			BBPunfix(b->batCacheid);
		if (g)
			BBPunfix(g->batCacheid);
		if (e)
			BBPunfix(e->batCacheid);
		throw(MAL, malfunc, RUNTIME_OBJECT_MISSING);
	}
	assert(BAThdense(b));
	assert(BAThdense(g));
	assert(BAThdense(e));
	assert(b->hseqbase == g->hseqbase);
	assert(BATcount(b) == BATcount(g));
	if (tp == TYPE_any && (grpfunc1 == BATgroupmedian || quantilefunc == BATgroupquantile))
		tp = b->ttype;
	if (grpfunc1)
		bn = (*grpfunc1)(b, g, e, NULL, tp, skip_nils, 1);
	if (quantilefunc) {
		assert(BATcount(quantile)>0);
		assert(quantile->ttype == TYPE_dbl);
		qvalue = ((const double *)Tloc(quantile, BUNfirst(quantile)))[0];
		if (qvalue <  0|| qvalue > 1) {
			char *s;
			s = createException(MAL, malfunc, "quantile value of %f is not in range [0,1]", qvalue);
			return s;
		}
		bn = (*quantilefunc)(b, g, e, NULL, tp, qvalue, skip_nils, 1);
		BBPunfix(quantile->batCacheid);
	}
	if (grpfunc2 && (*grpfunc2)(&bn, retval2 ? &cnts : NULL, b, g, e, NULL, tp, skip_nils, 1) != GDK_SUCCEED)
		bn = NULL;
	if (bn != NULL && (grpfunc1 == BATgroupmin || grpfunc1 == BATgroupmax)) {
		BAT *t = BATproject(bn, b);
		BBPunfix(bn->batCacheid);
		bn = t;
	}
	BBPunfix(b->batCacheid);
	BBPunfix(g->batCacheid);
	BBPunfix(e->batCacheid);
	if (bn == NULL) {
		char *errbuf = GDKerrbuf;
		char *s;

		if (errbuf && *errbuf) {
			if (strncmp(errbuf, "!ERROR: ", 8) == 0)
				errbuf += 8;
			if (strchr(errbuf, '!') == errbuf + 5) {
				s = createException(MAL, malfunc, "%s", errbuf);
			} else if ((s = strchr(errbuf, ':')) != NULL && s[1] == ' ') {
				s = createException(MAL, malfunc, "%s", s + 2);
			} else {
				s = createException(MAL, malfunc, "%s", errbuf);
			}
			*GDKerrbuf = 0;
			return s;
		}
		throw(MAL, malfunc, OPERATION_FAILED);
	}
	*retval1 = bn->batCacheid;
	BBPkeepref(bn->batCacheid);
	if (retval2) {
		*retval2 = cnts->batCacheid;
		BBPkeepref(cnts->batCacheid);
	}
	return MAL_SUCCEED;
}

static str
AGGRgrouped3(bat *retval1, bat *retval2, const bat *bid, const bat *gid, const bat *eid, int tp,
			 BAT *(*grpfunc1)(BAT *, BAT *, BAT *, BAT *, int, int, int),
			 gdk_return (*grpfunc2)(BAT **, BAT **, BAT *, BAT *, BAT *, BAT *, int, int, int),
			 int skip_nils,
			 const char *malfunc)
{
	BAT *b, *g, *e;

	b = BATdescriptor(*bid);	/* [head,value] */
	g = BATdescriptor(*gid);	/* [head,gid] */
	e = BATdescriptor(*eid);	/* [gid,any] */
	return AGGRgrouped(retval1, retval2, b, g, e, tp, grpfunc1, grpfunc2, NULL, 0, skip_nils, malfunc);
}

static str
AGGRgrouped2(bat *retval1, bat *retval2, const bat *bid, const bat *eid, int tp,
			 BAT *(*grpfunc1)(BAT *, BAT *, BAT *, BAT *, int, int, int),
			 gdk_return (*grpfunc2)(BAT **, BAT **, BAT *, BAT *, BAT *, BAT *, int, int, int),
			 int skip_nils,
			 const char *malfunc)
{
	BAT *b, *g, *e;

	b = BATdescriptor(*bid);	/* [gid,value] */
	if (b == NULL)
		throw(MAL, "aggr.sum", RUNTIME_OBJECT_MISSING);
	g = BATmirror(BATmark(b, 0)); /* [dense,gid] */
	e = BATmirror(BATmark(BATmirror(b), 0)); /* [dense,value] */
	BBPunfix(b->batCacheid);
	b = e;
	e = BATdescriptor(*eid);	/* [gid,any] */
	return AGGRgrouped(retval1, retval2, b, g, e, tp, grpfunc1, grpfunc2, NULL, 0, skip_nils, malfunc);
}

aggr_export str AGGRsum3_bte(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRsum3_bte(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_bte,
						BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRsum3_sht(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRsum3_sht(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_sht,
						BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRsum3_int(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRsum3_int(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_int,
						BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRsum3_wrd(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRsum3_wrd(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_wrd,
						BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRsum3_lng(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRsum3_lng(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_lng,
						BATgroupsum, NULL, 1, "aggr.sum");
}

#ifdef HAVE_HGE
aggr_export str AGGRsum3_hge(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRsum3_hge(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_hge,
						BATgroupsum, NULL, 1, "aggr.sum");
}
#endif

aggr_export str AGGRsum3_flt(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRsum3_flt(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_flt,
						BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRsum3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRsum3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_dbl,
						BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRsum2_bte(bat *retval, const bat *bid, const bat *eid);
str
AGGRsum2_bte(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_bte, BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRsum2_sht(bat *retval, const bat *bid, const bat *eid);
str
AGGRsum2_sht(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_sht, BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRsum2_int(bat *retval, const bat *bid, const bat *eid);
str
AGGRsum2_int(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_int, BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRsum2_wrd(bat *retval, const bat *bid, const bat *eid);
str
AGGRsum2_wrd(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_wrd, BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRsum2_lng(bat *retval, const bat *bid, const bat *eid);
str
AGGRsum2_lng(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_lng, BATgroupsum, NULL, 1, "aggr.sum");
}

#ifdef HAVE_HGE
aggr_export str AGGRsum2_hge(bat *retval, const bat *bid, const bat *eid);
str
AGGRsum2_hge(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_hge, BATgroupsum, NULL, 1, "aggr.sum");
}
#endif

aggr_export str AGGRsum2_flt(bat *retval, const bat *bid, const bat *eid);
str
AGGRsum2_flt(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_flt, BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRsum2_dbl(bat *retval, const bat *bid, const bat *eid);
str
AGGRsum2_dbl(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_dbl, BATgroupsum, NULL, 1, "aggr.sum");
}

aggr_export str AGGRprod3_bte(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRprod3_bte(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_bte,
						BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRprod3_sht(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRprod3_sht(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_sht,
						BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRprod3_int(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRprod3_int(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_int,
						BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRprod3_wrd(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRprod3_wrd(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_wrd,
						BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRprod3_lng(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRprod3_lng(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_lng,
						BATgroupprod, NULL, 1, "aggr.prod");
}

#ifdef HAVE_HGE
aggr_export str AGGRprod3_hge(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRprod3_hge(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_hge,
						BATgroupprod, NULL, 1, "aggr.prod");
}
#endif

aggr_export str AGGRprod3_flt(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRprod3_flt(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_flt,
						BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRprod3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRprod3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_dbl,
						BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRprod2_bte(bat *retval, const bat *bid, const bat *eid);
str
AGGRprod2_bte(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_bte, BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRprod2_sht(bat *retval, const bat *bid, const bat *eid);
str
AGGRprod2_sht(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_sht, BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRprod2_int(bat *retval, const bat *bid, const bat *eid);
str
AGGRprod2_int(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_int, BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRprod2_wrd(bat *retval, const bat *bid, const bat *eid);
str
AGGRprod2_wrd(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_wrd, BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRprod2_lng(bat *retval, const bat *bid, const bat *eid);
str
AGGRprod2_lng(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_lng, BATgroupprod, NULL, 1, "aggr.prod");
}

#ifdef HAVE_HGE
aggr_export str AGGRprod2_hge(bat *retval, const bat *bid, const bat *eid);
str
AGGRprod2_hge(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_hge, BATgroupprod, NULL, 1, "aggr.prod");
}
#endif

aggr_export str AGGRprod2_flt(bat *retval, const bat *bid, const bat *eid);
str
AGGRprod2_flt(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_flt, BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRprod2_dbl(bat *retval, const bat *bid, const bat *eid);
str
AGGRprod2_dbl(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_dbl, BATgroupprod, NULL, 1, "aggr.prod");
}

aggr_export str AGGRavg13_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRavg13_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_dbl,
						NULL, BATgroupavg, 1, "aggr.avg");
}

aggr_export str AGGRavg12_dbl(bat *retval, const bat *bid, const bat *eid);
str
AGGRavg12_dbl(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_dbl, NULL, BATgroupavg, 1, "aggr.avg");
}

aggr_export str AGGRavg23_dbl(bat *retval1, bat *retval2, const bat *bid, const bat *gid, const bat *eid);
str
AGGRavg23_dbl(bat *retval1, bat *retval2, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval1, retval2, bid, gid, eid, TYPE_dbl,
						NULL, BATgroupavg, 1, "aggr.avg");
}

aggr_export str AGGRavg22_dbl(bat *retval1, bat *retval2, const bat *bid, const bat *eid);
str
AGGRavg22_dbl(bat *retval1, bat *retval2, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval1, retval2, bid, eid, TYPE_dbl, NULL, BATgroupavg, 1, "aggr.avg");
}

aggr_export str AGGRstdev3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRstdev3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_dbl,
						BATgroupstdev_sample, NULL, 1, "aggr.stdev");
}

aggr_export str AGGRstdev2_dbl(bat *retval, const bat *bid, const bat *eid);
str
AGGRstdev2_dbl(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_dbl, BATgroupstdev_sample, NULL, 1, "aggr.stdev");
}

aggr_export str AGGRstdevp3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRstdevp3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_dbl,
						BATgroupstdev_population, NULL, 1, "aggr.stdevp");
}

aggr_export str AGGRstdevp2_dbl(bat *retval, const bat *bid, const bat *eid);
str
AGGRstdevp2_dbl(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_dbl, BATgroupstdev_population, NULL, 1, "aggr.stdevp");
}

aggr_export str AGGRvariance3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRvariance3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_dbl,
						BATgroupvariance_sample, NULL, 1, "aggr.variance");
}

aggr_export str AGGRvariance2_dbl(bat *retval, const bat *bid, const bat *eid);
str
AGGRvariance2_dbl(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_dbl, BATgroupvariance_sample, NULL, 1, "aggr.variance");
}

aggr_export str AGGRvariancep3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRvariancep3_dbl(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_dbl,
						BATgroupvariance_population, NULL, 1, "aggr.variancep");
}

aggr_export str AGGRvariancep2_dbl(bat *retval, const bat *bid, const bat *eid);
str
AGGRvariancep2_dbl(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_dbl, BATgroupvariance_population, NULL, 1, "aggr.variancep");
}

aggr_export str AGGRcount3(bat *retval, const bat *bid, const bat *gid, const bat *eid, const bit *ignorenils);
str
AGGRcount3(bat *retval, const bat *bid, const bat *gid, const bat *eid, const bit *ignorenils)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_wrd,
						BATgroupcount, NULL, *ignorenils, "aggr.count");
}

aggr_export str AGGRcount3nonils(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRcount3nonils(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_wrd,
						BATgroupcount, NULL, 1, "aggr.count");
}

aggr_export str AGGRcount3nils(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRcount3nils(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_wrd,
						BATgroupcount, NULL, 0, "aggr.count");
}

aggr_export str AGGRcount2(bat *retval, const bat *bid, const bat *eid, const bit *ignorenils);
str
AGGRcount2(bat *retval, const bat *bid, const bat *eid, const bit *ignorenils)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_wrd,
						BATgroupcount, NULL, *ignorenils, "aggr.count");
}

aggr_export str AGGRcount2nonils(bat *retval, const bat *bid, const bat *eid);
str
AGGRcount2nonils(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_wrd,
						BATgroupcount, NULL, 1, "aggr.count");
}

aggr_export str AGGRcount2nils(bat *retval, const bat *bid, const bat *eid);
str
AGGRcount2nils(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_wrd,
						BATgroupcount, NULL, 0, "aggr.count");
}

aggr_export str AGGRsize2nils(bat *retval, const bat *bid, const bat *eid);
str
AGGRsize2nils(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_wrd,
						BATgroupsize, NULL, 0, "aggr.size");
}

aggr_export str AGGRmin3(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRmin3(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_oid,
						BATgroupmin, NULL, 0, "aggr.min");
}

aggr_export str AGGRmin2(bat *retval, const bat *bid, const bat *eid);
str
AGGRmin2(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_oid, BATgroupmin, NULL, 0, "aggr.min");
}

aggr_export str AGGRmax3(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRmax3(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_oid,
						BATgroupmax, NULL, 0, "aggr.max");
}

aggr_export str AGGRmax2(bat *retval, const bat *bid, const bat *eid);
str
AGGRmax2(bat *retval, const bat *bid, const bat *eid)
{
	return AGGRgrouped2(retval, NULL, bid, eid, TYPE_oid, BATgroupmax, NULL, 0, "aggr.max");
}

aggr_export str AGGRmedian3(bat *retval, const bat *bid, const bat *gid, const bat *eid);
str
AGGRmedian3(bat *retval, const bat *bid, const bat *gid, const bat *eid)
{
	return AGGRgrouped3(retval, NULL, bid, gid, eid, TYPE_any,
						BATgroupmedian, NULL, 0, "aggr.median");
}


// XXX: when are these functions called?
aggr_export str AGGRquantile3(bat *retval, const bat *bid, const bat *gid, const bat *eid, const bat *quantile);
str
AGGRquantile3(bat *retval, const bat *bid, const bat *gid, const bat *eid, const bat *quantile)
{
	// this is inlined from AGGRgrouped3 to avoid changing all the other functions for now
	BAT *b, *g, *e, *q;
	b = BATdescriptor(*bid);	/* [head,value] */
	g = BATdescriptor(*gid);	/* [head,gid] */
	e = BATdescriptor(*eid);	/* [gid,any] */
	q = BATdescriptor(*quantile);
	return AGGRgrouped(retval, NULL, b, g, e, TYPE_any, NULL, NULL,
					   BATgroupquantile, q, 0, "aggr.quantile");
}

static str
AGGRsubgroupedExt(bat *retval1, bat *retval2, const bat *bid, const bat *gid, const bat *eid, const bat *sid,
			   int skip_nils, int abort_on_error, int tp,
			   BAT *(*grpfunc1)(BAT *, BAT *, BAT *, BAT *, int, int, int),
			   gdk_return (*grpfunc2)(BAT **, BAT **, BAT *, BAT *, BAT *, BAT *, int, int, int),
			   BAT *(*quantilefunc)(BAT *, BAT *, BAT *, BAT *, int, double, int, int),
			   const bat *quantile,
			   const char *malfunc)
{
	BAT *b, *g, *e, *s, *bn = NULL, *cnts, *q = NULL;
	double qvalue;

   /* one of grpfunc1, grpfunc2 and quantilefunc is non-NULL and the others are */
	assert((grpfunc1 && grpfunc2 == NULL && quantilefunc == NULL) ||
			(grpfunc1 == NULL && grpfunc2 && quantilefunc == NULL) ||
			(grpfunc1 == NULL && grpfunc2 == NULL && quantilefunc) );

	/* if retval2 is non-NULL, we must have grpfunc2 */
	assert(retval2 == NULL || grpfunc2 != NULL);

	b = BATdescriptor(*bid);
	g = gid ? BATdescriptor(*gid) : NULL;
	e = eid ? BATdescriptor(*eid) : NULL;
	q = quantile ? BATdescriptor(*quantile) : NULL;

	if (b == NULL || (gid != NULL && g == NULL) || (eid != NULL && e == NULL)) {
		if (b)
			BBPunfix(b->batCacheid);
		if (g)
			BBPunfix(g->batCacheid);
		if (e)
			BBPunfix(e->batCacheid);
		throw(MAL, malfunc, RUNTIME_OBJECT_MISSING);
	}
	if (tp == TYPE_any && (grpfunc1 == BATgroupmedian || quantilefunc == BATgroupquantile))
		tp = b->ttype;

	if (sid) {
		s = BATdescriptor(*sid);
		if (s == NULL) {
			BBPunfix(b->batCacheid);
			if (g)
				BBPunfix(g->batCacheid);
			if (e)
				BBPunfix(e->batCacheid);
			throw(MAL, malfunc, RUNTIME_OBJECT_MISSING);
		}
	} else {
		if (!BAThdense(b)) {
			/* XXX backward compatibility code: ignore non-dense head, but
			 * only if no candidate list */
			s = BATmirror(BATmark(BATmirror(b), 0));
			BBPunfix(b->batCacheid);
			b = s;
		}
		s = NULL;
	}
	if (grpfunc1)
		bn = (*grpfunc1)(b, g, e, s, tp, skip_nils, abort_on_error);
	if (quantilefunc) {
		assert(BATcount(q) > 0 || BATcount(b) == 0);
		assert(q->ttype == TYPE_dbl);
		if (BATcount(q) == 0) {
			qvalue = 0.5;
		} else {
			qvalue = ((const dbl *)Tloc(q, BUNfirst(q)))[0];
			if (qvalue <  0|| qvalue > 1) {
				char *s;
				s = createException(MAL, malfunc, "quantile value of %f is not in range [0,1]", qvalue);
				return s;
			}
		}
		bn = (*quantilefunc)(b, g, e, s, tp, qvalue, skip_nils, abort_on_error);
		BBPunfix(q->batCacheid);
	}
	if (grpfunc2 && (*grpfunc2)(&bn, retval2 ? &cnts : NULL, b, g, e, s, tp, skip_nils, abort_on_error) != GDK_SUCCEED)
		bn = NULL;

	BBPunfix(b->batCacheid);
	if (g)
		BBPunfix(g->batCacheid);
	if (e)
		BBPunfix(e->batCacheid);
	if (s)
		BBPunfix(s->batCacheid);
	if (bn == NULL) {
		char *errbuf = GDKerrbuf;
		char *s;

		if (errbuf && *errbuf) {
			if (strncmp(errbuf, "!ERROR: ", 8) == 0)
				errbuf += 8;
			if (strchr(errbuf, '!') == errbuf + 5) {
				s = createException(MAL, malfunc, "%s", errbuf);
			} else if ((s = strchr(errbuf, ':')) != NULL && s[1] == ' ') {
				s = createException(MAL, malfunc, "%s", s + 2);
			} else {
				s = createException(MAL, malfunc, "%s", errbuf);
			}
			*GDKerrbuf = 0;
			return s;
		}
		throw(MAL, malfunc, OPERATION_FAILED);
	}
	*retval1 = bn->batCacheid;
{'fraction_non_alphanumeric': 0.14863086118723431, 'fraction_numerical': 0.015881578663444317, 'mean_word_length': 4.607938044530494, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6176618', 'n_tokens_mistral': 20208, 'n_tokens_neox': 18904, 'n_words': 5504}