<commit_before>#include "signet/signet.h"
#include "check_signet.h"

START_TEST (check_signet_modification) 
{
	const char *phone1 = "1SOMENUMBER", *phone2 = "15124123529", *name1 = "check undef", *data1 = "undef data", *name2 = "check name", *data2 = "check check";
	size_t data_size;
	unsigned char *data;
	signet_t *signet;


	ck_assert_msg((signet = _signet_create(SIGNET_TYPE_ORG)) != NULL, "could not create signet.\n");

	ck_assert_msg(!(_signet_add_field_string(signet, SIGNET_ORG_UNDEFINED, name1, data1, 0)), "could not add undefined field.\n");

	ck_assert_msg((_signet_fid_exists(signet, SIGNET_ORG_UNDEFINED)) == 1, "could not find added undefined field.\n");

	ck_assert_msg((data = _signet_fetch_undef_name(signet, strlen(name1), (unsigned char *)name1, &data_size)) != NULL, "could not retrieve undefined field.\n");

	ck_assert_msg(!(memcmp(data, (unsigned char *)data1, data_size)), "undefined field was corrupted.\n");

	free(data);

	ck_assert_msg(!(_signet_add_field_string(signet, SIGNET_ORG_UNDEFINED, name2, data2, 0)), "could not add undefined field.\n");

	ck_assert_msg((data = _signet_fetch_undef_name(signet, strlen(name2), (unsigned char *)name2, &data_size)) != NULL, "could not retrieve undefined field.\n");

	ck_assert_msg(!(memcmp(data, (unsigned char *)data2, data_size)), "undefined field was corrupted.\n");

	free(data);

	ck_assert_msg(!(_signet_remove_undef_name(signet, strlen(name1), (unsigned char *)name1)), "could not remove undefined field by name.\n");

	ck_assert_msg((data = _signet_fetch_undef_name(signet, strlen(name2), (unsigned char *)name2, &data_size)) != NULL, "could not retrieve undefined field.\n");

	ck_assert_msg(!(memcmp(data, (unsigned char *)data2, data_size)), "undefined field was corrupted by the deletion of the previous field.\n");

	free(data);

	ck_assert_msg(!(_signet_add_field_string(signet, SIGNET_ORG_PHONE, NULL, phone1, 0)), "could not add phone field.\n");

	ck_assert_msg(_signet_fid_exists(signet, SIGNET_ORG_PHONE) == 1, "could not find added field.\n");

	ck_assert_msg((data = _signet_fetch_fid_num(signet, SIGNET_ORG_PHONE, 1, &data_size)) != NULL, "could not retrieve phone field.\n");

	ck_assert_msg(!(memcmp(data, (unsigned char *)phone1, data_size)), "field was corrupted.\n");

	free(data);

	ck_assert_msg(!(_signet_add_field_string(signet, SIGNET_ORG_PHONE, NULL, phone2, 0)), "could not add phone field.\n");

	ck_assert_msg((data = _signet_fetch_fid_num(signet, SIGNET_ORG_PHONE, 2, &data_size)) != NULL, "could not retrieve phone field.\n");

	ck_assert_msg(!(memcmp(data, (unsigned char *)phone2, data_size)), "field was corrupted.\n");

	free(data);

	ck_assert_msg(!(_signet_remove_fid_num(signet, SIGNET_ORG_PHONE, 1)), "could not remove field.\n");

	ck_assert_msg((data = _signet_fetch_fid_num(signet, SIGNET_ORG_PHONE, 1, &data_size)) != NULL, "could not retrieve phone field after previous field was deleted.\n");

	ck_assert_msg(!(memcmp(data, (unsigned char *)phone2, data_size)), "field was corrupted by the deletion of the previous field.\n");

	free(data);
	_signet_destroy(signet);
}
END_TEST

/*	to debug signet contents

	fprintf(stderr, "signet size: %d\nsignet data: ", signet->size);
	for(i = 0; i< signet->size; ++i)
		fprintf(stderr, "%d ", signet->data[i]);
	fprintf(stderr, "\nfield indexes: ");
	for(i = 0; i <= SIGNET_FID_MAX; ++i)
		fprintf(stderr, "%d ", signet->fields[i]);
	fprintf(stderr, "\n");
*/

START_TEST (check_signet_parsing)
{
	char *b64_sigone, *b64_sigtwo, *filename = "check.signet";
	uint32_t len;
	unsigned char *ser_sigone, *ser_sigtwo;
	signet_t *sigone, *sigtwo;

	sigone = _signet_create(SIGNET_TYPE_ORG);
	_signet_add_field_string(sigone, SIGNET_ORG_NAME, NULL, "some name", 0);
	_signet_add_field_string(sigone, SIGNET_ORG_PHONE, NULL, "phonenum1", 0);
	_signet_add_field_string(sigone, SIGNET_ORG_PHONE, NULL, "someotherphone", 0);
	_signet_add_field_string(sigone, SIGNET_ORG_UNDEFINED, "field name", "some field", 0);
	_signet_add_field_string(sigone, SIGNET_ORG_UNDEFINED, "check name", "check field", 0);
	_signet_add_field_string(sigone, SIGNET_ORG_UNDEFINED, "last field", "check check", 0);

	ck_assert_msg((ser_sigone = _signet_serialize(sigone, &len)) != NULL, "could not serialize signet to binary.\n");

	ck_assert_msg((sigtwo = _signet_deserialize(ser_sigone, len)) != NULL, "could not deserialize signet from binary.\n");

	ck_assert_msg((ser_sigtwo = _signet_serialize(sigtwo, &len)) != NULL, "could not serialize signet to binary the second time.\n");

	ck_assert_msg(!(memcmp(ser_sigone, ser_sigtwo, len)), "the signet got corrupted during serialization and deserialization.\n");

	free(ser_sigone);
	free(ser_sigtwo);
	_signet_destroy(sigtwo);

	ck_assert_msg((b64_sigone = _signet_serialize_b64(sigone)) != NULL, "could not serialize signet to base64.\n");
	
	ck_assert_msg((sigtwo = _signet_deserialize_b64(b64_sigone)) != NULL, "could not deserialize signet from base64.\n");

	ck_assert_msg((b64_sigtwo = _signet_serialize_b64(sigtwo)) != NULL, "could not serialize signet to base64 the second time.\n");

	ck_assert_msg(!(strcmp(b64_sigone, b64_sigtwo)), "the signet got corrupted during serialization and deserialization to and from b64.\n");

	free(b64_sigtwo);
	_signet_destroy(sigtwo);

	ck_assert_msg(!(_signet_to_file(sigone, filename)), "could not write signet to file.\n");<commit_msg>Manual re-merge with previous fixes.<commit_after>#include "signet/signet.h"
#include "check_signet.h"

START_TEST (check_signet_modification)
{
	const char *phone1 = "1SOMENUMBER", *phone2 = "15124123529", *name1 = "check undef", *data1 = "undef data", *name2 = "check name", *data2 = "check check";
	size_t data_size;
	unsigned char *data;
	signet_t *signet;


	ck_assert_msg((signet = _signet_create(SIGNET_TYPE_ORG)) != NULL, "could not create signet.\n");

	ck_assert_msg(!(_signet_add_field_string(signet, SIGNET_ORG_UNDEFINED, name1, data1, 0)), "could not add undefined field.\n");

	ck_assert_msg((_signet_fid_exists(signet, SIGNET_ORG_UNDEFINED)) == 1, "could not find added undefined field.\n");

	ck_assert_msg((data = _signet_fetch_undef_name(signet, strlen(name1), (unsigned char *)name1, &data_size)) != NULL, "could not retrieve undefined field.\n");

	ck_assert_msg(!(memcmp(data, (unsigned char *)data1, data_size)), "undefined field was corrupted.\n");

	free(data);

	ck_assert_msg(!(_signet_add_field_string(signet, SIGNET_ORG_UNDEFINED, name2, data2, 0)), "could not add undefined field.\n");

	ck_assert_msg((data = _signet_fetch_undef_name(signet, strlen(name2), (unsigned char *)name2, &data_size)) != NULL, "could not retrieve undefined field.\n");

	ck_assert_msg(!(memcmp(data, (unsigned char *)data2, data_size)), "undefined field was corrupted.\n");

	free(data);

	ck_assert_msg(!(_signet_remove_undef_name(signet, strlen(name1), (unsigned char *)name1)), "could not remove undefined field by name.\n");

	ck_assert_msg((data = _signet_fetch_undef_name(signet, strlen(name2), (unsigned char *)name2, &data_size)) != NULL, "could not retrieve undefined field.\n");

	ck_assert_msg(!(memcmp(data, (unsigned char *)data2, data_size)), "undefined field was corrupted by the deletion of the previous field.\n");

	free(data);

	ck_assert_msg(!(_signet_add_field_string(signet, SIGNET_ORG_PHONE, NULL, phone1, 0)), "could not add phone field.\n");

	ck_assert_msg(_signet_fid_exists(signet, SIGNET_ORG_PHONE) == 1, "could not find added field.\n");

	ck_assert_msg((data = _signet_fetch_fid_num(signet, SIGNET_ORG_PHONE, 1, &data_size)) != NULL, "could not retrieve phone field.\n");

	ck_assert_msg(!(memcmp(data, (unsigned char *)phone1, data_size)), "field was corrupted.\n");

	free(data);

	ck_assert_msg(!(_signet_add_field_string(signet, SIGNET_ORG_PHONE, NULL, phone2, 0)), "could not add phone field.\n");

	ck_assert_msg((data = _signet_fetch_fid_num(signet, SIGNET_ORG_PHONE, 2, &data_size)) != NULL, "could not retrieve phone field.\n");

	ck_assert_msg(!(memcmp(data, (unsigned char *)phone2, data_size)), "field was corrupted.\n");

	free(data);

	ck_assert_msg(!(_signet_remove_fid_num(signet, SIGNET_ORG_PHONE, 1)), "could not remove field.\n");

	ck_assert_msg((data = _signet_fetch_fid_num(signet, SIGNET_ORG_PHONE, 1, &data_size)) != NULL, "could not retrieve phone field after previous field was deleted.\n");

	ck_assert_msg(!(memcmp(data, (unsigned char *)phone2, data_size)), "field was corrupted by the deletion of the previous field.\n");

	free(data);
	_signet_destroy(signet);
}
END_TEST

/*	to debug signet contents

	fprintf(stderr, "signet size: %d\nsignet data: ", signet->size);
	for(i = 0; i< signet->size; ++i)
		fprintf(stderr, "%d ", signet->data[i]);
	fprintf(stderr, "\nfield indexes: ");
	for(i = 0; i <= SIGNET_FID_MAX; ++i)
		fprintf(stderr, "%d ", signet->fields[i]);
	fprintf(stderr, "\n");
*/

START_TEST (check_signet_parsing)
{
	char *b64_sigone, *b64_sigtwo, *filename = "check.signet";
	uint32_t len;
	unsigned char *ser_sigone, *ser_sigtwo;
	signet_t *sigone, *sigtwo;

	sigone = _signet_create(SIGNET_TYPE_ORG);
	_signet_add_field_string(sigone, SIGNET_ORG_NAME, NULL, "some name", 0);
	_signet_add_field_string(sigone, SIGNET_ORG_PHONE, NULL, "phonenum1", 0);
	_signet_add_field_string(sigone, SIGNET_ORG_PHONE, NULL, "someotherphone", 0);
	_signet_add_field_string(sigone, SIGNET_ORG_UNDEFINED, "field name", "some field", 0);
	_signet_add_field_string(sigone, SIGNET_ORG_UNDEFINED, "check name", "check field", 0);
	_signet_add_field_string(sigone, SIGNET_ORG_UNDEFINED, "last field", "check check", 0);

	ck_assert_msg((ser_sigone = _signet_serialize(sigone, &len)) != NULL, "could not serialize signet to binary.\n");

	ck_assert_msg((sigtwo = _signet_deserialize(ser_sigone, len)) != NULL, "could not deserialize signet from binary.\n");

	ck_assert_msg((ser_sigtwo = _signet_serialize(sigtwo, &len)) != NULL, "could not serialize signet to binary the second time.\n");

	ck_assert_msg(!(memcmp(ser_sigone, ser_sigtwo, len)), "the signet got corrupted during serialization and deserialization.\n");

	free(ser_sigone);
	free(ser_sigtwo);
	_signet_destroy(sigtwo);

	ck_assert_msg((b64_sigone = _signet_serialize_b64(sigone)) != NULL, "could not serialize signet to base64.\n");

	ck_assert_msg((sigtwo = _signet_deserialize_b64(b64_sigone)) != NULL, "could not deserialize signet from base64.\n");

	ck_assert_msg((b64_sigtwo = _signet_serialize_b64(sigtwo)) != NULL, "could not serialize signet to base64 the second time.\n");

	ck_assert_msg(!(strcmp(b64_sigone, b64_sigtwo)), "the signet got corrupted during serialization and deserialization to and from b64.\n");

	free(b64_sigtwo);
	_signet_destroy(sigtwo);

	ck_assert_msg(!(_signet_to_file(sigone, filename)), "could not write signet to file.\n");
{'fraction_non_alphanumeric': 0.1432424890707841, 'fraction_numerical': 0.01655659938610362, 'mean_word_length': 6.842450765864332, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7120263', 'n_tokens_mistral': 4175, 'n_tokens_neox': 3922, 'n_words': 905}