<commit_before>/**************************************************************************
* This code is licensed under the Apache License 2.0.  See ../LICENSE     *
* Copyright 2015 <NAME>                                             *
*                                                                         *
* data.h - defines data structures and format                             *
*                                                                         *
* Updates:                                                                *
**************************************************************************/



#ifndef MINISTRY_DATA_H
#define MINISTRY_DATA_H

// this is used in stats
#define TAGS_SEPARATOR			';'


// rounds the structure to 16k with a spare space
#define PTLIST_SIZE				2046

#define DHASH_CHECK_MOMENTS		0x01
#define DHASH_CHECK_MODE		0x02
#define DHASH_CHECK_PREDICT		0x04


enum data_conn_type
{
	DATA_TYPE_STATS = 0,
	DATA_TYPE_ADDER,
	DATA_TYPE_GAUGE,
	DATA_TYPE_HISTO,
	DATA_TYPE_COMPAT,
	DATA_TYPE_MAX
};

struct data_type_params
{
	uint8_t				type;
	const char		*	name;
	line_fn			*	lf;
	line_fn			*	pf;
	add_fn			*	af;
	dupd_fn			*	uf;
	buf_fn			*	bp;
	int16_t				tokn;
	uint16_t			port;
	int64_t				thrd;
	int8_t				styl;
	const char		*	sock;
	NET_TYPE		*	nt;
	ST_CFG			*	stc;
};

extern DTYPE data_type_defns[];


#define dp_set( _dp, t, v )			_dp.ts = t; _dp.val = v
#define dp_get_t( _dp )				_dp.ts
#define dp_get_v( _dp )				_dp.val

#define dpp_set( __dp, t, v )		__dp->ts = t; __dp->val = v
#define dpp_get_t( __dp )			__dp->ts
#define dpp_get_v( __dp )			__dp->val


#define dhash_do_moments( _d )		( _d->checks & DHASH_CHECK_MOMENTS )
#define dhash_do_mode( _d )			( _d->checks & DHASH_CHECK_MODE )
#define dhash_do_predict( _d )		( ( _d->checks & DHASH_CHECK_PREDICT ) && _d->predict )


struct data_point
{
	double				ts;
	double				val;
};


struct data_histogram	// size 16
{
	int64_t			*	counts;
	ST_HIST			*	conf;
};


struct points_list		// 8192
{
	PTLIST			*	next;
	int64_t				count;
	double				vals[PTLIST_SIZE];
};


struct data_hash_vals	// size 40
{
	PTLIST			*	points;
	DHIST				hist;
	double				total;
	int64_t				count;
};


struct data_hash_entry	// size 104
{
	DHASH			*	next;
	char			*	path;	// full path
	char			*	base;	// copy with the base part
	char			*	tags;	// does not hold memory - points into path

	// in.points is pre-allocated by the stats pass
	// we cannot assume in.points non-null means we
	// have data
	DVAL				in;
	DVAL				proc;

	// predictor structure, present or absent
	PRED			*	predict;

	dhash_lock_t	*	lock;

	uint16_t			sz;		// alloc'd size
	uint16_t			len;	// whole len
	uint16_t			blen;	// base len
	uint16_t			tlen;	// tags len

	uint64_t			sum;

	uint32_t			id;

	uint8_t				valid;
	uint8_t				do_pass;
	uint8_t				type;<commit_msg>removes duplicate points definition<commit_after>/**************************************************************************
* This code is licensed under the Apache License 2.0.  See ../LICENSE     *
* Copyright 2015 <NAME>                                             *
*                                                                         *
* data.h - defines data structures and format                             *
*                                                                         *
* Updates:                                                                *
**************************************************************************/



#ifndef MINISTRY_DATA_H
#define MINISTRY_DATA_H

// this is used in stats
#define TAGS_SEPARATOR			';'


// rounds the structure to 16k with a spare space
#define PTLIST_SIZE				2046

#define DHASH_CHECK_MOMENTS		0x01
#define DHASH_CHECK_MODE		0x02
#define DHASH_CHECK_PREDICT		0x04


enum data_conn_type
{
	DATA_TYPE_STATS = 0,
	DATA_TYPE_ADDER,
	DATA_TYPE_GAUGE,
	DATA_TYPE_HISTO,
	DATA_TYPE_COMPAT,
	DATA_TYPE_MAX
};

struct data_type_params
{
	uint8_t				type;
	const char		*	name;
	line_fn			*	lf;
	line_fn			*	pf;
	add_fn			*	af;
	dupd_fn			*	uf;
	buf_fn			*	bp;
	int16_t				tokn;
	uint16_t			port;
	int64_t				thrd;
	int8_t				styl;
	const char		*	sock;
	NET_TYPE		*	nt;
	ST_CFG			*	stc;
};

extern DTYPE data_type_defns[];


#define dp_set( _dp, t, v )			_dp.ts = t; _dp.val = v
#define dp_get_t( _dp )				_dp.ts
#define dp_get_v( _dp )				_dp.val

#define dpp_set( __dp, t, v )		__dp->ts = t; __dp->val = v
#define dpp_get_t( __dp )			__dp->ts
#define dpp_get_v( __dp )			__dp->val


#define dhash_do_moments( _d )		( _d->checks & DHASH_CHECK_MOMENTS )
#define dhash_do_mode( _d )			( _d->checks & DHASH_CHECK_MODE )
#define dhash_do_predict( _d )		( ( _d->checks & DHASH_CHECK_PREDICT ) && _d->predict )



struct data_histogram	// size 16
{
	int64_t			*	counts;
	ST_HIST			*	conf;
};


struct points_list		// 8192
{
	PTLIST			*	next;
	int64_t				count;
	double				vals[PTLIST_SIZE];
};


struct data_hash_vals	// size 40
{
	PTLIST			*	points;
	DHIST				hist;
	double				total;
	int64_t				count;
};


struct data_hash_entry	// size 104
{
	DHASH			*	next;
	char			*	path;	// full path
	char			*	base;	// copy with the base part
	char			*	tags;	// does not hold memory - points into path

	// in.points is pre-allocated by the stats pass
	// we cannot assume in.points non-null means we
	// have data
	DVAL				in;
	DVAL				proc;

	// predictor structure, present or absent
	PRED			*	predict;

	dhash_lock_t	*	lock;

	uint16_t			sz;		// alloc'd size
	uint16_t			len;	// whole len
	uint16_t			blen;	// base len
	uint16_t			tlen;	// tags len

	uint64_t			sum;

	uint8_t				valid;
	uint8_t				do_pass;
	uint8_t				type;
{'fraction_non_alphanumeric': 0.1369960756332501, 'fraction_numerical': 0.02176239743132358, 'mean_word_length': 2.130653266331658, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2939822', 'n_tokens_mistral': 2424, 'n_tokens_neox': 2141, 'n_words': 524}