<commit_before>/*
 * SysDB - src/core/data.c
 * Copyright (C) 2014 Sebastian 'tokkee' Harl <<EMAIL>>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#if HAVE_CONFIG_H
#	include "config.h"
#endif /* HAVE_CONFIG_H */

#include "sysdb.h"

#include "core/data.h"
#include "utils/error.h"

#include <assert.h>

#include <errno.h>

#include <inttypes.h>

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include <math.h>

/*
 * Operator support maxtrix.
 * <type1> <op> <type2> -> op_matrix[<op>][<type1>][<type2>]
 */

/* add, sub, mul, div, mod, concat */

/* integer, decimal, string, datetime, binary, regex */

static int op_matrix[6][6][6] = {
	/* SDB_DATA_ADD */
	{
		{ SDB_TYPE_INTEGER, -1, -1, -1, -1, -1 },
		{ -1, SDB_TYPE_DECIMAL, -1, -1, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ -1, -1, -1, SDB_TYPE_DATETIME, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
	},

	/* SDB_DATA_SUB */
	{
		{ SDB_TYPE_INTEGER, -1, -1, -1, -1, -1 },
		{ -1, SDB_TYPE_DECIMAL, -1, -1, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ -1, -1, -1, SDB_TYPE_DATETIME, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
	},

	/* SDB_DATA_MUL */
	{
		{ SDB_TYPE_INTEGER, -1, -1, SDB_TYPE_DATETIME, -1, -1 },
		{ -1, SDB_TYPE_DECIMAL, -1, SDB_TYPE_DATETIME, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ SDB_TYPE_DATETIME, SDB_TYPE_DATETIME, -1, SDB_TYPE_DATETIME, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
	},

	/* SDB_DATA_DIV */
	{
		{ SDB_TYPE_INTEGER, -1, -1, -1, -1, -1 },
		{ -1, SDB_TYPE_DECIMAL, -1, -1, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ SDB_TYPE_DATETIME, SDB_TYPE_DATETIME, -1, SDB_TYPE_DATETIME, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
	},

	/* SDB_DATA_MOD */
	{
		{ SDB_TYPE_INTEGER, -1, -1, -1, -1, -1 },
		{ -1, SDB_TYPE_DECIMAL, -1, -1, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ SDB_TYPE_DATETIME, SDB_TYPE_DATETIME, -1, SDB_TYPE_DATETIME, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
	},

	/* SDB_DATA_CONCAT */
	{
		{ -1, -1, -1, -1, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ -1, -1, SDB_TYPE_STRING, -1, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ -1, -1, -1, -1, SDB_TYPE_BINARY, -1 },
		{ -1, -1, -1, -1, -1, -1 },
	},
};

/*
 * private helper functions
 */

/* this function supports in-place copies */
static int
copy_array_values(sdb_data_t *dst, const sdb_data_t *src, size_t elem_size)
{
	int type = src->type & 0xff;

	if ((type == SDB_TYPE_INTEGER) || (type == SDB_TYPE_DECIMAL)) {
		if (dst != src)
			memcpy(dst->data.array.values, src->data.array.values,
					src->data.array.length * elem_size);
	}
	else if (type == SDB_TYPE_STRING) {
		char **s = src->data.array.values;
		char **d = dst->data.array.values;
		size_t i;

		for (i = 0; i < src->data.array.length; ++i) {
			d[i] = strdup(s[i]);
			if (! d[i])
				return -1;
		}
	}
	else {
		/* TODO */
		errno = ENOTSUP;
		return -1;
	}
	return 0;
} /* copy_array_values */

static void
free_array_values(sdb_data_t *datum)
{
	int type = datum->type & 0xff;

	if (type == SDB_TYPE_STRING) {
		char **v = datum->data.array.values;
		size_t i;

		for (i = 0; i < datum->data.array.length; ++i) {
			if (v[i])
				free(v[i]);
			v[i] = NULL;
		}
	}<commit_msg>data: Free binary and regex array values as well.<commit_after>/*
 * SysDB - src/core/data.c
 * Copyright (C) 2014 Sebastian 'tokkee' Harl <<EMAIL>>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#if HAVE_CONFIG_H
#	include "config.h"
#endif /* HAVE_CONFIG_H */

#include "sysdb.h"

#include "core/data.h"
#include "utils/error.h"

#include <assert.h>

#include <errno.h>

#include <inttypes.h>

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include <math.h>

/*
 * Operator support maxtrix.
 * <type1> <op> <type2> -> op_matrix[<op>][<type1>][<type2>]
 */

/* add, sub, mul, div, mod, concat */

/* integer, decimal, string, datetime, binary, regex */

static int op_matrix[6][6][6] = {
	/* SDB_DATA_ADD */
	{
		{ SDB_TYPE_INTEGER, -1, -1, -1, -1, -1 },
		{ -1, SDB_TYPE_DECIMAL, -1, -1, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ -1, -1, -1, SDB_TYPE_DATETIME, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
	},

	/* SDB_DATA_SUB */
	{
		{ SDB_TYPE_INTEGER, -1, -1, -1, -1, -1 },
		{ -1, SDB_TYPE_DECIMAL, -1, -1, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ -1, -1, -1, SDB_TYPE_DATETIME, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
	},

	/* SDB_DATA_MUL */
	{
		{ SDB_TYPE_INTEGER, -1, -1, SDB_TYPE_DATETIME, -1, -1 },
		{ -1, SDB_TYPE_DECIMAL, -1, SDB_TYPE_DATETIME, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ SDB_TYPE_DATETIME, SDB_TYPE_DATETIME, -1, SDB_TYPE_DATETIME, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
	},

	/* SDB_DATA_DIV */
	{
		{ SDB_TYPE_INTEGER, -1, -1, -1, -1, -1 },
		{ -1, SDB_TYPE_DECIMAL, -1, -1, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ SDB_TYPE_DATETIME, SDB_TYPE_DATETIME, -1, SDB_TYPE_DATETIME, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
	},

	/* SDB_DATA_MOD */
	{
		{ SDB_TYPE_INTEGER, -1, -1, -1, -1, -1 },
		{ -1, SDB_TYPE_DECIMAL, -1, -1, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ SDB_TYPE_DATETIME, SDB_TYPE_DATETIME, -1, SDB_TYPE_DATETIME, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
	},

	/* SDB_DATA_CONCAT */
	{
		{ -1, -1, -1, -1, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ -1, -1, SDB_TYPE_STRING, -1, -1, -1 },
		{ -1, -1, -1, -1, -1, -1 },
		{ -1, -1, -1, -1, SDB_TYPE_BINARY, -1 },
		{ -1, -1, -1, -1, -1, -1 },
	},
};

/*
 * private helper functions
 */

/* this function supports in-place copies */
static int
copy_array_values(sdb_data_t *dst, const sdb_data_t *src, size_t elem_size)
{
	int type = src->type & 0xff;

	if ((type == SDB_TYPE_INTEGER) || (type == SDB_TYPE_DECIMAL)) {
		if (dst != src)
			memcpy(dst->data.array.values, src->data.array.values,
					src->data.array.length * elem_size);
	}
	else if (type == SDB_TYPE_STRING) {
		char **s = src->data.array.values;
		char **d = dst->data.array.values;
		size_t i;

		for (i = 0; i < src->data.array.length; ++i) {
			d[i] = strdup(s[i]);
			if (! d[i])
				return -1;
		}
	}
	else {
		/* TODO */
		errno = ENOTSUP;
		return -1;
	}
	return 0;
} /* copy_array_values */

static void
free_array_values(sdb_data_t *datum)
{
	int type = datum->type & 0xff;

	if (type == SDB_TYPE_STRING) {
		char **v = datum->data.array.values;
		size_t i;

		for (i = 0; i < datum->data.array.length; ++i) {
			if (v[i])
				free(v[i]);
			v[i] = NULL;
		}
	}
	else if (type == SDB_TYPE_BINARY) {
		struct {
			size_t length;
			unsigned char *datum;
		} *v = datum->data.array.values;
		size_t i;

		for (i = 0; i < datum->data.array.length; ++i) {
			if (v[i].datum)
				free(v[i].datum);
			v[i].datum = NULL;
		}
	}
	else if (type == SDB_TYPE_REGEX) {
		struct {
			char *raw;
			regex_t regex;
		} *v = datum->data.array.values;
		size_t i;

		for (i = 0; i < datum->data.array.length; ++i) {
			if (v[i].raw) {
				free(v[i].raw);
				regfree(&v[i].regex);
			}
			v[i].raw = NULL;
		}
	}
{'fraction_non_alphanumeric': 0.19442436757873, 'fraction_numerical': 0.04377903975219411, 'mean_word_length': 2.6633888048411496, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 46, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4463135', 'n_tokens_mistral': 4489, 'n_tokens_neox': 4164, 'n_words': 1180}