<commit_before>/*
  Copyright (C) 2016 Eneo Tecnologia S.L.
  Author: <NAME> <<EMAIL>>
  Based on Luca Deri nprobe 6.22 collector

  This program is free software; you can redistribute it and/or modify
  it under the terms of the GNU Affero General Public License as
  published by the Free Software Foundation, either version 3 of the
  License, or (at your option) any later version.

  This program is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU Affero General Public License for more details.

  You should have received a copy of the GNU General Public License
  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "f2k.h"

#include "rb_netflow_test.h"

#include <setjmp.h>
#include <cmocka.h>

/// @todo template+flow in the same message
struct TestV9Template{
	V9FlowHeader flowHeader;
	V9TemplateHeader flowSetHeader;
	V9TemplateDef templateHeader;
	V9FlowSet templateSet[10];
};

struct TestV9Flow{
	V9FlowHeader flowHeader;
	V9TemplateHeader flowSetHeader;
	const uint8_t buffer1[72*3];
}__attribute__((packed));

static const struct TestV9Template v9Template = {
	.flowHeader = {
		/*uint16_t*/ .version = 0x0900,           /* Current version=9*/
		/*uint16_t*/ .count = 0x0100,           /* The number of records in PDU. */
		/*uint32_t*/ .sys_uptime = 0x00003039,     /* Current time in msecs since router booted */
		/*uint32_t*/ .unix_secs = 0xe2336552,     /* Current seconds since 0000 UTC 1970 */
		/*uint32_t*/ .flow_sequence = 0x38040000, /* Sequence number of total flows seen */
		/*uint32_t*/ .source_id = 0x01000000,      /* Source id */
	},

	.flowSetHeader = {
		/*uint16_t*/ .templateFlowset = 0x0000,
		/*uint16_t*/ .flowsetLen = 0x3000,
	},

	.templateHeader = {
		/*uint16_t*/ .templateId = 0x0301, /*259*/
		/*uint16_t*/ .fieldCount = 0x0a00,
	},

	.templateSet = { /* all uint16_t*/
		[0] = {.templateId = 0x6d01 /* 365: STA_MAC_ADDRESS  */, .flowsetLen = 0x0600},
		[1] = {.templateId = 0x6e01 /* 366: STA_IPV4_ADDRESS */, .flowsetLen = 0x0400},
		[2] = {.templateId = 0x5f00 /*  95: APPLICATION_ID */,   .flowsetLen = 0x0400},
		[3] = {.templateId = 0x9300 /* 147: WLAN_SSID */, .flowsetLen = 0x2100},
		[4] = {.templateId = 0x3d00 /*  61: DIRECTION */, .flowsetLen = 0x0100},
		[5] = {.templateId = 0x0100 /*   1: BYTES */, .flowsetLen = 0x0800},
		[6] = {.templateId = 0x0200 /*   2: PKTS */, .flowsetLen = 0x0800},
		[7] = {.templateId = 0x6200 /*  98: Not processed */, .flowsetLen = 0x0100},
		[8] = {.templateId = 0xc300 /* 195: Not processed */, .flowsetLen = 0x0100},
		[9] = {.templateId = 0x616f /* 367: WAP_MAC_ADDRESS */, .flowsetLen = 0x0600},
	}
};

static const struct TestV9Flow v9Flow = {
	.flowHeader = {
		/*uint16_t*/ .version = 0x0900,           /* Current version=9*/
		/*uint16_t*/ .count = 0x0100,           /* The number of records in PDU. */
		/*uint32_t*/ .sys_uptime = 0x00003039,     /* Current time in msecs since router booted */
		/*uint32_t*/ .unix_secs = 0x98346552,     /* Current seconds since 0000 UTC 1970 */
		/*uint32_t*/ .flow_sequence = 0x76040000, /* Sequence number of total flows seen */
		/*uint32_t*/ .source_id = 0x01000000,      /* Source id */
	},

	.flowSetHeader = {
		/*uint16_t*/ .templateFlowset = 0x0301,
		/*uint16_t*/ .flowsetLen = 0xd800,
	},

	.buffer1 = {
		0x00, 0x05, 0x69, 0x28, 0xb0, 0xc7, /* STA_MAC_ADDRESS: b8:14:c2:28:b0:c7 */
		0x0a, 0x0d, 0x5e, 0xdf,             /* STA_IPV4_ADDRESS: 10.13.94.223 */
		0x00, 0x00, 0x00, 0x00,             /* App Id: 0:0 */
		0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x2d, /* WLAN_SSID: "local-wifi" */
		0x77, 0x69, 0x66, 0x69, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00,
		0x00,                               /* Direction: Ingress */
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0xb3, /* Octetos */
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x07, /* Paquetes */
		0x00, 0x00,                         /* Not used */
		0x58, 0xbf, 0xea, 0x01, 0x5b, 0x40, /* WAP_MAC_ADDRESS */

		/* ****************************** */

		0x00, 0x05, 0x69, 0x28, 0xb0, 0xc7, /* STA_MAC_ADDRESS: b8:14:c2:28:b0:c7 */
		0x0a, 0x0d, 0x5e, 0xdf,             /* STA_IPV4_ADDRESS: 10.13.94.223 */
		0x0d, 0x00, 0x01, 0xc5,             /* App Id: 13:453 */
		0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x2d, /* WLAN_SSID: "local-wifi" */
		0x77, 0x69, 0x66, 0x69, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00,
		0x00,                               /* Direction: Ingress */
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0xb3, /* Octetos */
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x07, /* Paquetes */
		0x00, 0x00,                         /* Not used */
		0x58, 0xbf, 0xea, 0x01, 0x5b, 0x40, /* WAP_MAC_ADDRESS */

		/* ****************************** */

		0x00, 0x05, 0x69, 0x28, 0xb0, 0xc7, /* STA_MAC_ADDRESS: b8:14:c2:28:b0:c7 */
		0x0a, 0x0d, 0x5e, 0xdf,             /* STA_IPV4_ADDRESS: 10.13.94.223 */
		0x03, 0x00, 0x00, 0x35,             /* App Id: 3:53 */
		0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x2d, /* WLAN_SSID: "local-wifi" */
		0x77, 0x69, 0x66, 0x69, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00,
		0x00,                               /* Direction: Ingress */
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, 0xb3, /* Octetos */
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x07, /* Paquetes */
		0x00, 0x00,                         /* Not used */
		0x58, 0xbf, 0xea, 0x01, 0x5b, 0x40, /* WAP_MAC_ADDRESS */
	},
};

static const struct checkdata_value checkdata1[] = {
	{.key = "type", .value="netflowv9"},
	{.key = "application_id", .value=NULL},
	{.key = "application_id_name", .value=NULL},
	{.key = "engine_id", .value=NULL},
	{.key = "engine_id_name", .value=NULL},
};

static const struct checkdata_value checkdata2[] = {
	{.key="type", .value="netflowv9"},
	{.key="application_id", .value=NULL},
	{.key="application_id_name", .value="ssl"},
	{.key="engine_id", .value=NULL},
	{.key="engine_id_name", .value="PANA-L7"},
};

static const struct checkdata_value checkdata3[] = {
	{.key = "type", .value="netflowv9"},
	{.key = "application_id", .value=NULL},
	{.key = "application_id_name", .value="dns"},
	{.key = "engine_id", .value=NULL},
	{.key = "engine_id_name", .value="IANA-L4"},
};

static int prepare_test_nf9_appid_enrichment(void **state) {
	static const struct checkdata sl1_checkdata[] = {
		{.size = RD_ARRAYSIZE(checkdata1), .checks = checkdata1},
		{.size = RD_ARRAYSIZE(checkdata2), .checks = checkdata2},
		{.size = RD_ARRAYSIZE(checkdata3), .checks = checkdata3},
	};

#define TEST(config_path, mhosts_db_path, mrecord, mrecord_size, checks,       \
								checks_size) { \
		.config_json_path = config_path,                               \
		.host_list_path = mhosts_db_path,                              \
		.netflow_src_ip = 0x04030201,                                  \
		.record = mrecord, .record_size = mrecord_size,                \
		.checkdata = checks, .checkdata_size = checks_size             \
	}

	struct test_params test_params[] = {
		[0] = TEST("./tests/0000-testFlowV5.json", "./tests/0010-data/",
				&v9Template, sizeof(v9Template),
				NULL, 0),

		[1] = TEST(NULL, NULL, &v9Flow, sizeof(v9Flow),
			sl1_checkdata, RD_ARRAYSIZE(sl1_checkdata)),
	};
<commit_msg>Use of netflow_meta in tests/0010-testNFv9AppIdEnrichment<commit_after>/*
  Copyright (C) 2016 Eneo Tecnologia S.L.
  Author: <NAME> <<EMAIL>>
  Based on Luca Deri nprobe 6.22 collector

  This program is free software; you can redistribute it and/or modify
  it under the terms of the GNU Affero General Public License as
  published by the Free Software Foundation, either version 3 of the
  License, or (at your option) any later version.

  This program is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU Affero General Public License for more details.

  You should have received a copy of the GNU General Public License
  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

#include "f2k.h"

#include "rb_netflow_test.h"

#include <setjmp.h>
#include <cmocka.h>

#define WLAN_SSID_CHARS \
	0x6c, 0x6f, 0x63, 0x61, 0x6c, 0x2d, /* WLAN_SSID: "local-wifi" */ \
	0x77, 0x69, 0x66, 0x69, 0x00, 0x00, \
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, \
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, \
	0x00, 0x00, 0x00, 0x00, 0x00, 0x00, \
	0x00, 0x00, 0x00

#define APP_ID_ENTITIES(RT, R) \
	RT(STA_MAC_ADDRESS, 6, 0, 0x00, 0x05, 0x69, 0x28, 0xb0, 0xc7) \
	RT(STA_IPV4_ADDRESS, 4, 0, 10, 13, 94, 223) \
	RT(APPLICATION_ID, 4, 0, UINT32_TO_UINT8_ARR(0)) \
	RT(WLAN_SSID, 33, 0,  WLAN_SSID_CHARS) \
	RT(DIRECTION, 1, 0, 0) \
	RT(IN_BYTES, 8, 0, UINT64_TO_UINT8_ARR(7603)) \
	RT(IN_PKTS, 8, 0, UINT64_TO_UINT8_ARR(263)) \
	RT(98, 1, 0, 0) \
	RT(195, 1, 0, 0) \
	RT(WAP_MAC_ADDRESS, 6, 0, 0x58, 0xbf, 0xea, 0x01, 0x5b, 0x40) \
		/* ****************************** */ \
	R(STA_MAC_ADDRESS, 6, 0, 0x00, 0x05, 0x69, 0x28, 0xb0, 0xc7) \
	R(STA_IPV4_ADDRESS, 4, 0, 10, 13, 94, 223) \
	R(APPLICATION_ID, 4, 0, FLOW_APPLICATION_ID(13, 453)) \
	R(WLAN_SSID, 33, 0,  WLAN_SSID_CHARS) \
	R(DIRECTION, 1, 0, 0) \
	R(BYTES, 8, 0, UINT64_TO_UINT8_ARR(7603)) \
	R(PKTS, 8, 0, UINT64_TO_UINT8_ARR(263)) \
	R(98, 1, 0, 0) \
	R(195, 1, 0, 0) \
	R(WAP_MAC_ADDRESS, 6, 0, 0x58, 0xbf, 0xea, 0x01, 0x5b, 0x40) \
		/* ****************************** */ \
	R(STA_MAC_ADDRESS, 6, 0, 0x00, 0x05, 0x69, 0x28, 0xb0, 0xc7) \
	R(STA_IPV4_ADDRESS, 4, 0, 10, 13, 94, 223) \
	R(APPLICATION_ID, 4, 0, FLOW_APPLICATION_ID(3, 53)) \
	R(WLAN_SSID, 33, 0,  WLAN_SSID_CHARS) \
	R(DIRECTION, 1, 0, 0) \
	R(BYTES, 8, 0, UINT64_TO_UINT8_ARR(7603)) \
	R(PKTS, 8, 0, UINT64_TO_UINT8_ARR(263)) \
	R(98, 1, 0, 0) \
	R(195, 1, 0, 0) \
	R(WAP_MAC_ADDRESS, 6, 0, 0x58, 0xbf, 0xea, 0x01, 0x5b, 0x40)

#define TEST_TEMPLATE_ID 1025

#define TEST_FLOW_HEADER \
	.unix_secs = constexpr_be32toh(1467220140), \
	.flow_sequence = constexpr_be32toh(12372811), \

static const struct checkdata_value checkdata1[] = {
	{.key = "type", .value="netflowv9"},
	{.key = "application_id", .value=NULL},
	{.key = "application_id_name", .value=NULL},
	{.key = "engine_id", .value=NULL},
	{.key = "engine_id_name", .value=NULL},
};

static const struct checkdata_value checkdata2[] = {
	{.key="type", .value="netflowv9"},
	{.key="application_id", .value=NULL},
	{.key="application_id_name", .value="ssl"},
	{.key="engine_id", .value=NULL},
	{.key="engine_id_name", .value="PANA-L7"},
};

static const struct checkdata_value checkdata3[] = {
	{.key = "type", .value="netflowv9"},
	{.key = "application_id", .value=NULL},
	{.key = "application_id_name", .value="dns"},
	{.key = "engine_id", .value=NULL},
	{.key = "engine_id_name", .value="IANA-L4"},
};

static int prepare_test_nf9_appid_enrichment(void **state) {
	static const NF9_TEMPLATE(template, TEST_FLOW_HEADER,
		TEST_TEMPLATE_ID, APP_ID_ENTITIES);
	static const NF9_FLOW(flow, TEST_FLOW_HEADER, TEST_TEMPLATE_ID,
		APP_ID_ENTITIES);

	static const struct checkdata sl1_checkdata[] = {
		{.size = RD_ARRAYSIZE(checkdata1), .checks = checkdata1},
		{.size = RD_ARRAYSIZE(checkdata2), .checks = checkdata2},
		{.size = RD_ARRAYSIZE(checkdata3), .checks = checkdata3},
	};

#define TEST(config_path, mhosts_db_path, mrecord, mrecord_size, checks,       \
								checks_size) { \
		.config_json_path = config_path,                               \
		.host_list_path = mhosts_db_path,                              \
		.netflow_src_ip = 0x04030201,                                  \
		.record = mrecord, .record_size = mrecord_size,                \
		.checkdata = checks, .checkdata_size = checks_size             \
	}

	struct test_params test_params[] = {
		[0] = TEST("./tests/0000-testFlowV5.json", "./tests/0010-data/",
				&template, sizeof(template),
				NULL, 0),

		[1] = TEST(NULL, NULL, &flow, sizeof(flow),
			sl1_checkdata, RD_ARRAYSIZE(sl1_checkdata)),
	};

{'fraction_non_alphanumeric': 0.16201891715590347, 'fraction_numerical': 0.11945531637312459, 'mean_word_length': 3.154810298102981, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 3, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2348349', 'n_tokens_mistral': 6269, 'n_tokens_neox': 5142, 'n_words': 1200}