<commit_before>#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include <error.h>
#include "dict.h"
#include "quid.h"
#include "zmalloc.h"
#include "marshall.h"
#include "dict_marshall.h"

/*
 * Recursively parse dictionary into marshall object
 */
marshall_t *marshall_dict_decode(char *data, size_t data_len, char *name, size_t name_len, void *parent) {
	dict_parser p;
	dict_token_t t[data_len];

	dict_init(&p);
	int o = dict_parse(&p, data, data_len, t, data_len);
	if (o < 1)
		return NULL;

	marshall_t *obj = (marshall_t *)tree_zcalloc(1, sizeof(marshall_t), parent);
	obj->child = (marshall_t **)tree_zcalloc(o, sizeof(marshall_t *), obj);

	if (name && name_len) {
		obj->name = name;
		obj->name_len = name_len;
	}

	switch (t[0].type) {
		case DICT_ARRAY: {
			int i;
			obj->type = MTYPE_ARRAY;
			for (i = 1; i < o; ++i) {
				switch (t[i].type) {
					case DICT_PRIMITIVE:
						obj->child[obj->size] = (marshall_t *)tree_zcalloc(1, sizeof(marshall_t), obj);
						if (dict_cmp(data, &t[i], "null")) {
							obj->child[obj->size]->type = MTYPE_NULL;
						} else if (dict_cmp(data, &t[i], "true")) {
							obj->child[obj->size]->type = MTYPE_TRUE;
						} else if (dict_cmp(data, &t[i], "false")) {
							obj->child[obj->size]->type = MTYPE_FALSE;
						} else {
							obj->child[obj->size]->type = MTYPE_INT;
							obj->child[obj->size]->data = tree_zstrndup(data + t[i].start, t[i].end - t[i].start, obj);
							obj->child[obj->size]->data_len = t[i].end - t[i].start;
							if (strismatch(obj->child[obj->size]->data, "-1234567890.")) {
								if (strccnt(data, '.') == 1) {
									obj->child[obj->size]->type = MTYPE_FLOAT;
								}
							}
						}
						obj->child[obj->size]->size = 1;
						obj->size++;
						break;
					case DICT_STRING:
						obj->child[obj->size] = (marshall_t *)tree_zcalloc(1, sizeof(marshall_t), obj);
						obj->child[obj->size]->type = MTYPE_STRING;
						obj->child[obj->size]->data = tree_zstrndup(data + t[i].start, t[i].end - t[i].start, obj);
						obj->child[obj->size]->data_len = t[i].end - t[i].start;
						if (strquid_format(obj->child[obj->size]->data) > 0)
							obj->child[obj->size]->type = MTYPE_QUID;
						obj->child[obj->size]->size = 1;
						obj->size++;
						break;
					case DICT_OBJECT: {
						obj->child[obj->size] = marshall_dict_decode(data + t[i].start, t[i].end - t[i].start, NULL, 0, obj);
						int x, j = 0;
						for (x = 0; x < t[i].size; x++) {
							j += dict_levelcount(&t[i + 1 + j], 0, 0, NULL);
							j += dict_levelcount(&t[i + 1 + j], 0, 0, NULL);
						}
						i += j;
						obj->size++;
						break;
					}
					case DICT_ARRAY: {
						obj->child[obj->size] = marshall_dict_decode(data + t[i].start, t[i].end - t[i].start, NULL, 0, obj);
						int x, j = 0;
						for (x = 0; x < t[i].size; x++) {
							j += dict_levelcount(&t[i + 1 + j], 0, 0, NULL);
						}
						i += j;
						obj->size++;
						break;
					}
					default:
						error_throw("70bef771b0a3", "Invalid datatype");
						break;
				}
			}
			break;
		}
		case DICT_OBJECT: {
			int i;
			obj->type = MTYPE_OBJECT;
			unsigned char setname = 0;
			for (i = 1; i < o; ++i) {
				switch (t[i].type) {
					case DICT_PRIMITIVE:
						if (dict_cmp(data, &t[i], "null")) {
							obj->child[obj->size]->type = MTYPE_NULL;
						} else if (dict_cmp(data, &t[i], "true")) {
							obj->child[obj->size]->type = MTYPE_TRUE;
						} else if (dict_cmp(data, &t[i], "false")) {
							obj->child[obj->size]->type = MTYPE_FALSE;
						} else {
							obj->child[obj->size]->type = MTYPE_INT;
							obj->child[obj->size]->data = tree_zstrndup(data + t[i].start, t[i].end - t[i].start, obj);
							obj->child[obj->size]->data_len = t[i].end - t[i].start;
							if (strismatch(obj->child[obj->size]->data, "-1234567890.")) {
								if (strccnt(data, '.') == 1) {
									obj->child[obj->size]->type = MTYPE_FLOAT;
								}
							}
						}
						obj->child[obj->size]->size = 1;
						obj->size++;
						setname = 0;
						break;
					case DICT_STRING:
						if (!setname) {
							obj->child[obj->size] = (marshall_t *)tree_zcalloc(1, sizeof(marshall_t), obj);
							obj->child[obj->size]->type = MTYPE_STRING;
							obj->child[obj->size]->name = tree_zstrndup(data + t[i].start, t[i].end - t[i].start, obj);
							obj->child[obj->size]->name_len = t[i].end - t[i].start;
							setname = 1;
						} else {
							obj->child[obj->size]->data = tree_zstrndup(data + t[i].start, t[i].end - t[i].start, obj);
							obj->child[obj->size]->data_len = t[i].end - t[i].start;
							if (strquid_format(obj->child[obj->size]->data) > 0)
								obj->child[obj->size]->type = MTYPE_QUID;
							obj->child[obj->size]->size = 1;
							obj->size++;
							setname = 0;
						}
						break;
					case DICT_OBJECT: {
						obj->child[obj->size] = marshall_dict_decode(data + t[i].start, t[i].end - t[i].start, obj->child[obj->size]->name, obj->child[obj->size]->name_len, obj);<commit_msg>Fix: wrong FLOAT detection<commit_after>#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include <error.h>
#include "dict.h"
#include "quid.h"
#include "zmalloc.h"
#include "marshall.h"
#include "dict_marshall.h"

/*
 * Recursively parse dictionary into marshall object
 */
marshall_t *marshall_dict_decode(char *data, size_t data_len, char *name, size_t name_len, void *parent) {
	dict_parser p;
	dict_token_t t[data_len];

	dict_init(&p);
	int o = dict_parse(&p, data, data_len, t, data_len);
	if (o < 1)
		return NULL;

	marshall_t *obj = (marshall_t *)tree_zcalloc(1, sizeof(marshall_t), parent);
	obj->child = (marshall_t **)tree_zcalloc(o, sizeof(marshall_t *), obj);

	if (name && name_len) {
		obj->name = name;
		obj->name_len = name_len;
	}

	switch (t[0].type) {
		case DICT_ARRAY: {
			int i;
			obj->type = MTYPE_ARRAY;
			for (i = 1; i < o; ++i) {
				switch (t[i].type) {
					case DICT_PRIMITIVE:
						obj->child[obj->size] = (marshall_t *)tree_zcalloc(1, sizeof(marshall_t), obj);
						if (dict_cmp(data, &t[i], "null")) {
							obj->child[obj->size]->type = MTYPE_NULL;
						} else if (dict_cmp(data, &t[i], "true")) {
							obj->child[obj->size]->type = MTYPE_TRUE;
						} else if (dict_cmp(data, &t[i], "false")) {
							obj->child[obj->size]->type = MTYPE_FALSE;
						} else {
							obj->child[obj->size]->type = MTYPE_INT;
							obj->child[obj->size]->data = tree_zstrndup(data + t[i].start, t[i].end - t[i].start, obj);
							obj->child[obj->size]->data_len = t[i].end - t[i].start;
							if (strismatch(obj->child[obj->size]->data, "-1234567890.")) {
								if (strccnt(obj->child[obj->size]->data, '.') == 1) {
									obj->child[obj->size]->type = MTYPE_FLOAT;
								}
							}
						}
						obj->child[obj->size]->size = 1;
						obj->size++;
						break;
					case DICT_STRING:
						obj->child[obj->size] = (marshall_t *)tree_zcalloc(1, sizeof(marshall_t), obj);
						obj->child[obj->size]->type = MTYPE_STRING;
						obj->child[obj->size]->data = tree_zstrndup(data + t[i].start, t[i].end - t[i].start, obj);
						obj->child[obj->size]->data_len = t[i].end - t[i].start;
						if (strquid_format(obj->child[obj->size]->data) > 0)
							obj->child[obj->size]->type = MTYPE_QUID;
						obj->child[obj->size]->size = 1;
						obj->size++;
						break;
					case DICT_OBJECT: {
						obj->child[obj->size] = marshall_dict_decode(data + t[i].start, t[i].end - t[i].start, NULL, 0, obj);
						int x, j = 0;
						for (x = 0; x < t[i].size; x++) {
							j += dict_levelcount(&t[i + 1 + j], 0, 0, NULL);
							j += dict_levelcount(&t[i + 1 + j], 0, 0, NULL);
						}
						i += j;
						obj->size++;
						break;
					}
					case DICT_ARRAY: {
						obj->child[obj->size] = marshall_dict_decode(data + t[i].start, t[i].end - t[i].start, NULL, 0, obj);
						int x, j = 0;
						for (x = 0; x < t[i].size; x++) {
							j += dict_levelcount(&t[i + 1 + j], 0, 0, NULL);
						}
						i += j;
						obj->size++;
						break;
					}
					default:
						error_throw("70bef771b0a3", "Invalid datatype");
						break;
				}
			}
			break;
		}
		case DICT_OBJECT: {
			int i;
			obj->type = MTYPE_OBJECT;
			unsigned char setname = 0;
			for (i = 1; i < o; ++i) {
				switch (t[i].type) {
					case DICT_PRIMITIVE:
						if (dict_cmp(data, &t[i], "null")) {
							obj->child[obj->size]->type = MTYPE_NULL;
						} else if (dict_cmp(data, &t[i], "true")) {
							obj->child[obj->size]->type = MTYPE_TRUE;
						} else if (dict_cmp(data, &t[i], "false")) {
							obj->child[obj->size]->type = MTYPE_FALSE;
						} else {
							obj->child[obj->size]->type = MTYPE_INT;
							obj->child[obj->size]->data = tree_zstrndup(data + t[i].start, t[i].end - t[i].start, obj);
							obj->child[obj->size]->data_len = t[i].end - t[i].start;
							if (strismatch(obj->child[obj->size]->data, "-1234567890.")) {
								if (strccnt(obj->child[obj->size]->data, '.') == 1) {
									obj->child[obj->size]->type = MTYPE_FLOAT;
								}
							}
						}
						obj->child[obj->size]->size = 1;
						obj->size++;
						setname = 0;
						break;
					case DICT_STRING:
						if (!setname) {
							obj->child[obj->size] = (marshall_t *)tree_zcalloc(1, sizeof(marshall_t), obj);
							obj->child[obj->size]->type = MTYPE_STRING;
							obj->child[obj->size]->name = tree_zstrndup(data + t[i].start, t[i].end - t[i].start, obj);
							obj->child[obj->size]->name_len = t[i].end - t[i].start;
							setname = 1;
						} else {
							obj->child[obj->size]->data = tree_zstrndup(data + t[i].start, t[i].end - t[i].start, obj);
							obj->child[obj->size]->data_len = t[i].end - t[i].start;
							if (strquid_format(obj->child[obj->size]->data) > 0)
								obj->child[obj->size]->type = MTYPE_QUID;
							obj->child[obj->size]->size = 1;
							obj->size++;
							setname = 0;
						}
						break;
					case DICT_OBJECT: {
						obj->child[obj->size] = marshall_dict_decode(data + t[i].start, t[i].end - t[i].start, obj->child[obj->size]->name, obj->child[obj->size]->name_len, obj);
{'fraction_non_alphanumeric': 0.20660991487230845, 'fraction_numerical': 0.012418627941912869, 'mean_word_length': 2.615496017378711, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 271, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4583712', 'n_tokens_mistral': 5095, 'n_tokens_neox': 3906, 'n_words': 718}