<commit_before>#include <string.h>
#include <stdlib.h>
#include <stdbool.h>
#include <stdint.h>

#include <fxdiv.h>

#include <nnpack.h>
#include <nnpack/macros.h>
#include <nnpack/utils.h>
#include <nnpack/system.h>
#include <nnpack/hwinfo.h>
#include <nnpack/simd.h>

#include <nnpack/validation.h>
#include <nnpack/transform.h>

enum nnp_status nnp_convolution_inference(
	enum nnp_convolution_algorithm algorithm,
	enum nnp_convolution_transform_strategy transform_strategy,
	size_t input_channels,
	size_t output_channels,
	struct nnp_size input_size,
	struct nnp_padding input_padding,
	struct nnp_size kernel_size,
	const float input_pointer[],
	const float kernel_pointer[],
	const float bias[],
	float output_pointer[],
	pthreadpool_t threadpool,
	struct nnp_profile* profile)
{
	void* memory_block = NULL;
	NNP_TOTAL_START(profile)

	/* Basic validation of parameters. This check detects invalid, but not unsupported parameters. */
	enum nnp_status status = validate_convolution_arguments(
		1, input_channels, output_channels,
		input_size, input_padding, kernel_size);
	if (status != nnp_status_success) {
		goto cleanup;
	}

	const struct nnp_size output_size = {
		.width = input_padding.left + input_size.width + input_padding.right - kernel_size.width + 1,
		.height = input_padding.top + input_size.height + input_padding.bottom - kernel_size.height + 1
	};

	if (algorithm == nnp_convolution_algorithm_auto) {
		if (max(kernel_size.width, kernel_size.height) > 8) {
			algorithm = nnp_convolution_algorithm_ft16x16;
		} else {
			const size_t tile_count_8x8 =
				divide_round_up(output_size.height, 8 - kernel_size.height + 1) *
				divide_round_up(output_size.width, 8 - kernel_size.width + 1);
			const size_t tile_count_16x16 =
				divide_round_up(output_size.height, 16 - kernel_size.height + 1) *
				divide_round_up(output_size.width, 16 - kernel_size.width + 1);
			if (tile_count_8x8 <= 4 * tile_count_16x16) {
				/* 8x8 tiles are more efficient */
				if ((kernel_size.height == 3) && (kernel_size.width == 3)) {
					algorithm = nnp_convolution_algorithm_wt8x8;
				} else {
					algorithm = nnp_convolution_algorithm_ft8x8;
				}
			} else {
				algorithm = nnp_convolution_algorithm_ft16x16;
			}
		}
	}

	const size_t simd_width = nnp_hwinfo.simd_width;
	struct nnp_size tile_size;
	bool fourier_transform;
	nnp_transform_2d input_transform_function = NULL;
	nnp_transform_2d kernel_transform_function = NULL;
	nnp_transform_2d_with_bias output_transform_function = NULL;
	nnp_blockmac block_mac_function = NULL;
	switch (algorithm) {
		case nnp_convolution_algorithm_wt8x8:
			if ((kernel_size.height != 3) || (kernel_size.width != 3)) {
				status = nnp_status_unsupported_algorithm;
				goto cleanup;
			}
			tile_size = (struct nnp_size) { .height = 8, .width = 8 };
			input_transform_function = nnp_hwinfo.transforms.iwt_f6x6_3x3_and_stream;
			kernel_transform_function = nnp_hwinfo.transforms.kwt_f6x6_3x3;
			output_transform_function = nnp_hwinfo.transforms.owt_f6x6_3x3_with_bias;
			fourier_transform = false;
			break;
		case nnp_convolution_algorithm_ft8x8:
			tile_size = (struct nnp_size) { .height = 8, .width = 8 };
			input_transform_function = nnp_hwinfo.transforms.fft8x8_and_stream;
			kernel_transform_function = nnp_hwinfo.transforms.fft8x8_and_stream;
			output_transform_function = nnp_hwinfo.transforms.ifft8x8_with_bias;
			fourier_transform = true;
			break;
		case nnp_convolution_algorithm_ft16x16:
			tile_size = (struct nnp_size) { .height = 16, .width = 16 };
			input_transform_function = nnp_hwinfo.transforms.fft16x16_and_stream;
			kernel_transform_function = nnp_hwinfo.transforms.fft16x16_and_stream;
			output_transform_function = nnp_hwinfo.transforms.ifft16x16_with_bias;
			fourier_transform = true;
			break;
		case nnp_convolution_algorithm_auto:
			NNP_UNREACHABLE;
		default:
			status = nnp_status_invalid_algorithm;
			goto cleanup;
	}

	const size_t tuple_elements = (fourier_transform ? simd_width * 2 : simd_width);
	const size_t tuple_size = tuple_elements * sizeof(float);
	const size_t tile_elements = tile_size.height * tile_size.width;

	const struct nnp_size input_tile = {
		.width = tile_size.width,
		.height = tile_size.height
	};

	const struct nnp_size output_tile = {
		.width = input_tile.width - kernel_size.width + 1,
		.height = input_tile.height - kernel_size.height + 1
	};

	const size_t tiles_y_count = divide_round_up(output_size.height, output_tile.height);
	const struct fxdiv_divisor_size_t tiles_x_count = fxdiv_init_size_t(
		divide_round_up(output_size.width, output_tile.width));
	const size_t tiles_count = tiles_x_count.value * tiles_y_count;

	/* Calculate cache blocking parameters */
	const size_t cache_elements_l1 = nnp_hwinfo.blocking.l1 / (tuple_elements * sizeof(float));
	const size_t cache_elements_l2 = nnp_hwinfo.blocking.l2 / (tuple_elements * sizeof(float));
	const size_t cache_elements_l3 = nnp_hwinfo.blocking.l3 / (tuple_elements * sizeof(float));

	const size_t tiles_subblock_max = (fourier_transform ? nnp_hwinfo.cxgemm.mr : nnp_hwinfo.sxgemm.mr);
	const size_t output_channels_subblock_max = (fourier_transform ? nnp_hwinfo.cxgemm.nr : nnp_hwinfo.sxgemm.nr);

	const size_t input_channels_block_max =
		round_down(cache_elements_l1 / (tiles_subblock_max + output_channels_subblock_max), 2);
	const size_t tiles_block_max =
		round_down(cache_elements_l2 / input_channels_block_max, tiles_subblock_max);
	const size_t output_channels_block_max =
		round_down(cache_elements_l3 / input_channels_block_max, output_channels_subblock_max);

	const size_t transform_tile_size = tile_elements * sizeof(float);
	const size_t input_transform_size = tiles_count * min(input_channels, input_channels_block_max) * transform_tile_size;
	const size_t kernel_transform_size = output_channels * min(input_channels, input_channels_block_max) * transform_tile_size;
	const size_t output_transform_size = tiles_count * output_channels * transform_tile_size;
	const size_t memory_size = input_transform_size + kernel_transform_size + output_transform_size;

	memory_block = allocate_memory(memory_size);
	if (memory_block == NULL) {
		status = nnp_status_out_of_memory;
		goto cleanup;
	}

	float* input_transform = memory_block;
	float* kernel_transform = memory_block + input_transform_size;
	float* output_transform = memory_block + input_transform_size + kernel_transform_size;

	{
		const float (*input)[input_size.width][input_size.height] =
			(const float(*)[input_size.width][input_size.height]) input_pointer;
		const float (*kernel)[input_channels][kernel_size.width * kernel_size.height] =
			(const float(*)[input_channels][kernel_size.width * kernel_size.height]) kernel_pointer;
		float (*output)[output_size.width][output_size.height] =
			(float(*)[output_size.width][output_size.height]) output_pointer;

		switch (transform_strategy) {
			case nnp_convolution_transform_strategy_tuple_based:
				for (size_t input_channels_block_start = 0; input_channels_block_start < input_channels; input_channels_block_start += input_channels_block_max) {
					const size_t input_channels_block_size = min(input_channels - input_channels_block_start, input_channels_block_max);
<commit_msg>Convolution inference: fix crash bug<commit_after>#include <string.h>
#include <stdlib.h>
#include <stdbool.h>
#include <stdint.h>

#include <fxdiv.h>

#include <nnpack.h>
#include <nnpack/macros.h>
#include <nnpack/utils.h>
#include <nnpack/system.h>
#include <nnpack/hwinfo.h>
#include <nnpack/simd.h>

#include <nnpack/validation.h>
#include <nnpack/transform.h>

enum nnp_status nnp_convolution_inference(
	enum nnp_convolution_algorithm algorithm,
	enum nnp_convolution_transform_strategy transform_strategy,
	size_t input_channels,
	size_t output_channels,
	struct nnp_size input_size,
	struct nnp_padding input_padding,
	struct nnp_size kernel_size,
	const float input_pointer[],
	const float kernel_pointer[],
	const float bias[],
	float output_pointer[],
	pthreadpool_t threadpool,
	struct nnp_profile* profile)
{
	void* memory_block = NULL;
	NNP_TOTAL_START(profile)

	/* Basic validation of parameters. This check detects invalid, but not unsupported parameters. */
	enum nnp_status status = validate_convolution_arguments(
		1, input_channels, output_channels,
		input_size, input_padding, kernel_size);
	if (status != nnp_status_success) {
		goto cleanup;
	}

	const struct nnp_size output_size = {
		.width = input_padding.left + input_size.width + input_padding.right - kernel_size.width + 1,
		.height = input_padding.top + input_size.height + input_padding.bottom - kernel_size.height + 1
	};

	if (algorithm == nnp_convolution_algorithm_auto) {
		if (max(kernel_size.width, kernel_size.height) > 8) {
			algorithm = nnp_convolution_algorithm_ft16x16;
		} else {
			const size_t tile_count_8x8 =
				divide_round_up(output_size.height, 8 - kernel_size.height + 1) *
				divide_round_up(output_size.width, 8 - kernel_size.width + 1);
			const size_t tile_count_16x16 =
				divide_round_up(output_size.height, 16 - kernel_size.height + 1) *
				divide_round_up(output_size.width, 16 - kernel_size.width + 1);
			if (tile_count_8x8 <= 4 * tile_count_16x16) {
				/* 8x8 tiles are more efficient */
				if ((kernel_size.height == 3) && (kernel_size.width == 3)) {
					algorithm = nnp_convolution_algorithm_wt8x8;
				} else {
					algorithm = nnp_convolution_algorithm_ft8x8;
				}
			} else {
				algorithm = nnp_convolution_algorithm_ft16x16;
			}
		}
	}

	const size_t simd_width = nnp_hwinfo.simd_width;
	struct nnp_size tile_size;
	bool fourier_transform;
	nnp_transform_2d input_transform_function = NULL;
	nnp_transform_2d kernel_transform_function = NULL;
	nnp_transform_2d_with_bias output_transform_function = NULL;
	nnp_blockmac block_mac_function = NULL;
	switch (algorithm) {
		case nnp_convolution_algorithm_wt8x8:
			if ((kernel_size.height != 3) || (kernel_size.width != 3)) {
				status = nnp_status_unsupported_algorithm;
				goto cleanup;
			}
			tile_size = (struct nnp_size) { .height = 8, .width = 8 };
			input_transform_function = nnp_hwinfo.transforms.iwt_f6x6_3x3_and_stream;
			kernel_transform_function = nnp_hwinfo.transforms.kwt_f6x6_3x3;
			output_transform_function = nnp_hwinfo.transforms.owt_f6x6_3x3_with_bias;
			fourier_transform = false;
			break;
		case nnp_convolution_algorithm_ft8x8:
			tile_size = (struct nnp_size) { .height = 8, .width = 8 };
			input_transform_function = nnp_hwinfo.transforms.fft8x8_and_stream;
			kernel_transform_function = nnp_hwinfo.transforms.fft8x8_and_stream;
			output_transform_function = nnp_hwinfo.transforms.ifft8x8_with_bias;
			fourier_transform = true;
			break;
		case nnp_convolution_algorithm_ft16x16:
			tile_size = (struct nnp_size) { .height = 16, .width = 16 };
			input_transform_function = nnp_hwinfo.transforms.fft16x16_and_stream;
			kernel_transform_function = nnp_hwinfo.transforms.fft16x16_and_stream;
			output_transform_function = nnp_hwinfo.transforms.ifft16x16_with_bias;
			fourier_transform = true;
			break;
		case nnp_convolution_algorithm_auto:
			NNP_UNREACHABLE;
		default:
			status = nnp_status_invalid_algorithm;
			goto cleanup;
	}

	const size_t tuple_elements = (fourier_transform ? simd_width * 2 : simd_width);
	const size_t tuple_size = tuple_elements * sizeof(float);
	const size_t tile_elements = tile_size.height * tile_size.width;

	const struct nnp_size input_tile = {
		.width = tile_size.width,
		.height = tile_size.height
	};

	const struct nnp_size output_tile = {
		.width = input_tile.width - kernel_size.width + 1,
		.height = input_tile.height - kernel_size.height + 1
	};

	const size_t tiles_y_count = divide_round_up(output_size.height, output_tile.height);
	const struct fxdiv_divisor_size_t tiles_x_count = fxdiv_init_size_t(
		divide_round_up(output_size.width, output_tile.width));
	const size_t tiles_count = tiles_x_count.value * tiles_y_count;

	/* Calculate cache blocking parameters */
	const size_t cache_elements_l1 = nnp_hwinfo.blocking.l1 / (tuple_elements * sizeof(float));
	const size_t cache_elements_l2 = nnp_hwinfo.blocking.l2 / (tuple_elements * sizeof(float));
	const size_t cache_elements_l3 = nnp_hwinfo.blocking.l3 / (tuple_elements * sizeof(float));

	const size_t tiles_subblock_max = (fourier_transform ? nnp_hwinfo.cxgemm.mr : nnp_hwinfo.sxgemm.mr);
	const size_t output_channels_subblock_max = (fourier_transform ? nnp_hwinfo.cxgemm.nr : nnp_hwinfo.sxgemm.nr);

	const size_t input_channels_block_max =
		round_down(cache_elements_l1 / (tiles_subblock_max + output_channels_subblock_max), 2);
	const size_t tiles_block_max =
		round_down(cache_elements_l2 / input_channels_block_max, tiles_subblock_max);
	const size_t output_channels_block_max =
		round_down(cache_elements_l3 / input_channels_block_max, output_channels_subblock_max);

	const size_t transform_tile_size = tile_elements * sizeof(float);
	const size_t input_transform_size = tiles_count * min(input_channels, input_channels_block_max) * transform_tile_size;
	const size_t kernel_transform_size = output_channels * min(input_channels, input_channels_block_max) * transform_tile_size;
	const size_t output_transform_size = tiles_count * output_channels * transform_tile_size;
	const size_t memory_size = input_transform_size + kernel_transform_size + output_transform_size;

	memory_block = allocate_memory(memory_size);
	if (memory_block == NULL) {
		status = nnp_status_out_of_memory;
		goto cleanup;
	}

	float* input_transform = memory_block;
	float* kernel_transform = memory_block + input_transform_size;
	float* output_transform = memory_block + input_transform_size + kernel_transform_size;

	{
		const float (*input)[input_size.height][input_size.width] =
			(const float(*)[input_size.height][input_size.width]) input_pointer;
		const float (*kernel)[input_channels][kernel_size.width * kernel_size.height] =
			(const float(*)[input_channels][kernel_size.width * kernel_size.height]) kernel_pointer;
		float (*output)[output_size.height][output_size.width] =
			(float(*)[output_size.height][output_size.width]) output_pointer;

		switch (transform_strategy) {
			case nnp_convolution_transform_strategy_tuple_based:
				for (size_t input_channels_block_start = 0; input_channels_block_start < input_channels; input_channels_block_start += input_channels_block_max) {
					const size_t input_channels_block_size = min(input_channels - input_channels_block_start, input_channels_block_max);

{'fraction_non_alphanumeric': 0.08766346087317511, 'fraction_numerical': 0.014944994118868056, 'mean_word_length': 5.694766095414543, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1851836', 'n_tokens_mistral': 5476, 'n_tokens_neox': 5038, 'n_words': 912}