<commit_before>/**************************************************************************
* This code is licensed under the Apache License 2.0.  See ../LICENSE     *
* Copyright 2015 <NAME>                                             *
*                                                                         *
* data/json.c - handling functions for json data                          *
*                                                                         *
* Updates:                                                                *
**************************************************************************/


#include "local.h"


int data_parse_json_element( DTYPE *dt, json_object *el )
{
	json_object *po, *vo, *ve;
	const char *path, *sval;
	enum json_type jt;
	int plen, is_gg;
	size_t i, vlen;
	double val;
	DHASH *d;
	char op;

	// look for the path
	if( !( po = json_object_object_get( el, "path" ) )
	 || !json_object_is_type( po, json_type_string ) )
		return -1;

	// then values or points
	if( ( !( vo = json_object_object_get( el, "values" ) )
	   && !( vo = json_object_object_get( el, "points" ) ) ) 
	   || !json_object_is_type( vo, json_type_array ) )
		return 1;

	path = json_object_get_string( po );
	plen = json_object_get_string_len( po );

	if( !( d = data_get_dhash( (char *) path, plen, dt->stc ) ) )
		return 1;

	vlen = json_object_array_length( vo );

	// no values?  OK... but why?
	if( !vlen )
		return 0;

	// is this a gauge type?  We need to 
	// check the sign and pass that in if so
	is_gg = ( dt->type == DATA_TYPE_GAUGE );

	for( i = 0; i < vlen; i++ )
	{
		if( !( ve = json_object_array_get_idx( vo, i ) ) )
			continue;

		jt = json_object_get_type( ve );
		op = '\0';

		// if you put something other than strings
		switch( jt )
		{
			case json_type_string:
				sval = json_object_get_string( ve );
				if( is_gg )
				{
					if( *sval == '+' || *sval == '-' )
						op = *sval++;
				}

				val = strtod( sval, NULL );
				break;

			case json_type_double:
				val = json_object_get_double( ve );
				break;

			default:
				//info( "No type." );
				continue;
		}

		//info( "[%s] Update %s with value %f, op %c.",
		//	dt->name, d->path, val, op );

		// call the update function
		(*(dt->uf))( d, val, op );
	}

	return 0;
}


/*
 * We expect
[
	{
		"path": "<path1>",
		"values": [ <list of doubles or strings> ]
	},
	{
		"path": "<path2>",
		"values": [ <list of doubles or strings> ]
	},
	...
]
 */

int data_parse_json( json_object *jo, DTYPE *dt )
{
	size_t i, pl, ret = 0;
	json_object *el;

	if( !jo )
	{
		warn( "No json object from json post?!" );
		return -1;
	}

	// we expect an array
	if( !json_object_is_type( jo, json_type_array ) )
		return -1;

	pl = json_object_array_length( jo );

	for( i = 0; i < pl; ++i )
	{
		if( !( el = json_object_array_get_idx( jo, i ) ) )
			continue;

		ret += data_parse_json_element( dt, el );
	}

	return ret;
}

void data_fetch_jcb( void *arg, json_object *jo )
{
	FETCH *f = (FETCH *) arg;
	data_parse_json( jo, f->dtype );
}<commit_msg>adds support for graphite json output format<commit_after>/**************************************************************************
* This code is licensed under the Apache License 2.0.  See ../LICENSE     *
* Copyright 2015 <NAME>                                             *
*                                                                         *
* data/json.c - handling functions for json data                          *
*                                                                         *
* Updates:                                                                *
**************************************************************************/


#include "local.h"


int data_parse_json_element( DTYPE *dt, json_object *el )
{
	json_object *po, *vo, *ve, *va;
	const char *path, *sval;
	enum json_type jt;
	int plen, is_gg;
	size_t i, vlen;
	double val;
	DHASH *d;
	char op;

	// look for the path
	if( ( !( po = json_object_object_get( el, "path" ) )
	   && !( po = json_object_object_get( el, "target" ) ) )
	   || !json_object_is_type( po, json_type_string ) )
		return -1;

	// then values or points
	if( ( !( vo = json_object_object_get( el, "values" ) )
	   && !( vo = json_object_object_get( el, "datapoints" ) ) )
	   || !json_object_is_type( vo, json_type_array ) )
		return 1;

	path = json_object_get_string( po );
	plen = json_object_get_string_len( po );

	// reject paths with parentheses - they have functions in
	if( memchr( path, '(', plen ) )
		return 1;

	if( !( d = data_get_dhash( (char *) path, plen, dt->stc ) ) )
		return 1;

	vlen = json_object_array_length( vo );

	// no values?  OK... but why?
	if( !vlen )
		return 0;

	// is this a gauge type?  We need to 
	// check the sign and pass that in if so
	is_gg = ( dt->type == DATA_TYPE_GAUGE );

	for( i = 0; i < vlen; i++ )
	{
		if( !( ve = json_object_array_get_idx( vo, i ) ) )
			continue;

		jt = json_object_get_type( ve );
		op = '\0';

		// if you put something other than strings
		switch( jt )
		{
			case json_type_string:
				sval = json_object_get_string( ve );
				if( is_gg )
				{
					if( *sval == '+' || *sval == '-' )
						op = *sval++;
				}

				val = strtod( sval, NULL );
				break;

			case json_type_double:
				val = json_object_get_double( ve );
				break;

			case json_type_array:
				// graphite format
				if( json_object_array_length( ve ) != 2 )
					continue;

				// format is value, timestamp
				// also, null happens
				if( !( va = json_object_array_get_idx( ve, 0 ) ) )
					continue;

				val = json_object_get_double( va );
				break;

			default:
				//info( "No type." );
				continue;
		}

		//info( "[%s] Update %s with value %f, op %c.",
		//	dt->name, d->path, val, op );

		// check val is valid
		if( !isnormal( val ) )
			continue;

		// call the update function
		(*(dt->uf))( d, val, op );
	}

	return 0;
}


/*
 * We expect
[
	{
		"path": "<path1>",
		"values": [ <list of doubles or strings> ]
	},
	{
		"path": "<path2>",
		"values": [ <list of doubles or strings> ]
	},
	...
]
 */

int data_parse_json( json_object *jo, DTYPE *dt )
{
	size_t i, pl, ret = 0;
	json_object *el;

	if( !jo )
	{
		warn( "No json object from json post?!" );
		return -1;
	}

	// we expect an array
	if( !json_object_is_type( jo, json_type_array ) )
		return -1;

	pl = json_object_array_length( jo );

	for( i = 0; i < pl; ++i )
	{
		if( !( el = json_object_array_get_idx( jo, i ) ) )
			continue;

		ret += data_parse_json_element( dt, el );
	}

	return ret;
}

void data_fetch_jcb( void *arg, json_object *jo )
{
	FETCH *f = (FETCH *) arg;
	data_parse_json( jo, f->dtype );
}

{'fraction_non_alphanumeric': 0.18218745298630962, 'fraction_numerical': 0.006168196178727246, 'mean_word_length': 2.1447492904446546, 'pattern_counts': {'":': 8, '<': 17, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1970422', 'n_tokens_mistral': 2619, 'n_tokens_neox': 2327, 'n_words': 649}