<commit_before>/**************************************************************************
* This code is licensed under the Apache License 2.0.  See ../LICENSE     *
* Copyright 2015 <NAME>                                             *
*                                                                         *
* stats.c - statistics functions and config                               *
*                                                                         *
* Updates:                                                                *
**************************************************************************/

#include "ministry.h"



const char *stats_type_names[STATS_TYPE_MAX] =
{
	"stats", "adder", "gauge", "self"
};



static inline int cmp_floats( const void *p1, const void *p2 )
{
	float *f1, *f2;

	f1 = (float *) p1;
	f2 = (float *) p2;

	return ( *f1 > *f2 ) ? 1  :
	       ( *f2 < *f1 ) ? -1 : 0;
}

// this macro is some serious va args abuse
#define bprintf( fmt, ... )			if( b->len > IO_BUF_HWMK ) \
									{ io_buf_send( b ); b = mem_new_buf( IO_BUF_SZ ); } \
									b->len += snprintf( b->buf + b->len, b->sz - b->len, "%s" fmt " %ld\n", prfx, ## __VA_ARGS__, ts )



int stats_report_one( DHASH *d, ST_THR *t, time_t ts, IOBUF **buf )
{
	PTLIST *list, *p;
	int i, ct, idx;
	ST_THOLD *thr;
	char *prfx;
	float sum;
	IOBUF *b;

	// grab the points list
	list = d->proc.points;
	d->proc.points = NULL;

	// count the points
	for( ct = 0, p = list; p; p = p->next )
		ct += p->count;

	// anything to do?
	if( ct == 0 )
	{
		if( list )
			mem_free_point_list( list );
		return 0;
	}

    b    = *buf;
	prfx = ctl->stats->stats->prefix;


	// if we have just one points structure, we just use
	// it's own vals array as our workspace.  We need to
	// sort in place, but only this fn holds that space
	// now, so that's ok.  If we have more than one, we
	// need a flat array, so make sure the workbuf is big
	// enough and copy each vals array into it

	if( list->next == NULL )
		t->wkspc = list->vals;
	else
	{
		// do we have enough workspace?
		if( ct > t->wkspcsz )
		{
			// double it until we have enough
			while( ct > t->wkspcsz )
				t->wkspcsz *= 2;

			// free the existing and grab a new chunk
			free( t->wkbuf );
			t->wkbuf = (float *) allocz( t->wkspcsz * sizeof( float ) );

			// make sure
			if( !t->wkbuf )
			{
				fatal( "Could not allocate new workbuf of %d floats.", t->wkspcsz );
				return 0;
			}
		}

		// and the workspace is the buffer
		t->wkspc = t->wkbuf;

		// now copy the data in
		for( i = 0, p = list; p; p = p->next )
		{
			memcpy( t->wkspc + i, p->vals, p->count * sizeof( float ) );
			i += p->count;
		}
	}

	sum = 0;
	kahan_summation( t->wkspc, ct, &sum );

	// median offset<commit_msg>Don't change the workspace until we get a new one<commit_after>/**************************************************************************
* This code is licensed under the Apache License 2.0.  See ../LICENSE     *
* Copyright 2015 <NAME>                                             *
*                                                                         *
* stats.c - statistics functions and config                               *
*                                                                         *
* Updates:                                                                *
**************************************************************************/

#include "ministry.h"



const char *stats_type_names[STATS_TYPE_MAX] =
{
	"stats", "adder", "gauge", "self"
};



static inline int cmp_floats( const void *p1, const void *p2 )
{
	float *f1, *f2;

	f1 = (float *) p1;
	f2 = (float *) p2;

	return ( *f1 > *f2 ) ? 1  :
	       ( *f2 < *f1 ) ? -1 : 0;
}

// this macro is some serious va args abuse
#define bprintf( fmt, ... )			if( b->len > IO_BUF_HWMK ) \
									{ io_buf_send( b ); b = mem_new_buf( IO_BUF_SZ ); } \
									b->len += snprintf( b->buf + b->len, b->sz - b->len, "%s" fmt " %ld\n", prfx, ## __VA_ARGS__, ts )



int stats_report_one( DHASH *d, ST_THR *t, time_t ts, IOBUF **buf )
{
	PTLIST *list, *p;
	int i, ct, idx;
	ST_THOLD *thr;
	char *prfx;
	float sum;
	IOBUF *b;

	// grab the points list
	list = d->proc.points;
	d->proc.points = NULL;

	// count the points
	for( ct = 0, p = list; p; p = p->next )
		ct += p->count;

	// anything to do?
	if( ct == 0 )
	{
		if( list )
			mem_free_point_list( list );
		return 0;
	}

    b    = *buf;
	prfx = ctl->stats->stats->prefix;


	// if we have just one points structure, we just use
	// it's own vals array as our workspace.  We need to
	// sort in place, but only this fn holds that space
	// now, so that's ok.  If we have more than one, we
	// need a flat array, so make sure the workbuf is big
	// enough and copy each vals array into it

	if( list->next == NULL )
		t->wkspc = list->vals;
	else
	{
		// do we have enough workspace?
		if( ct > t->wkspcsz )
		{
			int sz = t->wkspcsz;
			float *ws;

			// double it until we have enough
			while( ct > sz )
				sz *= 2;

			if( !( ws = (float *) allocz( sz * sizeof( float ) ) ) )
			{
				fatal( "Could not allocate new workbuf of %d floats.", sz );
				return 0;
			}

			// free the existing and grab a new chunk
			free( t->wkbuf );
			t->wkspcsz = sz;
			t->wkbuf   = ws;
		}

		// and the workspace is the buffer
		t->wkspc = t->wkbuf;

		// now copy the data in
		for( i = 0, p = list; p; p = p->next )
		{
			memcpy( t->wkspc + i, p->vals, p->count * sizeof( float ) );
			i += p->count;
		}
	}

	sum = 0;
	kahan_summation( t->wkspc, ct, &sum );

	// median offset
{'fraction_non_alphanumeric': 0.18714080459770116, 'fraction_numerical': 0.010057471264367816, 'mean_word_length': 1.8240365111561867, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 74, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1486430', 'n_tokens_mistral': 2142, 'n_tokens_neox': 1865, 'n_words': 624}