<commit_before>#include <stddef.h>
#include <stdint.h>

#include <nnpack/macros.h>
#include <nnpack/arm_neon.h>


void nnp_conv1x1_only_4x4__neon(
	size_t input_channels,
	size_t image_size,
	const float* input,
	const float* kernel,
	float* output)
{
	const float* input0 = input;
	const float* input1 = input0 + image_size;
	const float* input2 = input1 + image_size;
	const float* input3 = input2 + image_size;

	const float32x4_t vkernel0x = vld1q_f32(kernel);
	kernel += input_channels;
	const float32x4_t vkernel1x = vld1q_f32(kernel);
	kernel += input_channels;
	const float32x4_t vkernel2x = vld1q_f32(kernel);
	kernel += input_channels;
	const float32x4_t vkernel3x = vld1q_f32(kernel);

	float* output0 = output;
	float* output1 = output0 + image_size;
	float* output2 = output1 + image_size;
	float* output3 = output2 + image_size;
	while (image_size >= 4) {
		float32x4_t voutput0 = vld1q_f32(output0);
		float32x4_t voutput1 = vld1q_f32(output1);
		float32x4_t voutput2 = vld1q_f32(output2);
		float32x4_t voutput3 = vld1q_f32(output3);

		const float32x4_t vinput0 = vld1q_f32(input0); input0 += 4;
		#if defined(__aarch64__)
			voutput0 = vfmaq_lane_f32(voutput0, vinput0, vget_low_f32(vkernel0x), 0);
			voutput1 = vfmaq_lane_f32(voutput1, vinput0, vget_low_f32(vkernel1x), 0);
			voutput2 = vfmaq_lane_f32(voutput2, vinput0, vget_low_f32(vkernel2x), 0);
			voutput3 = vfmaq_lane_f32(voutput3, vinput0, vget_low_f32(vkernel3x), 0);
		#else
			voutput0 = vmlaq_lane_f32(voutput0, vinput0, vget_low_f32(vkernel0x), 0);
			voutput1 = vmlaq_lane_f32(voutput1, vinput0, vget_low_f32(vkernel1x), 0);
			voutput2 = vmlaq_lane_f32(voutput2, vinput0, vget_low_f32(vkernel2x), 0);
			voutput3 = vmlaq_lane_f32(voutput3, vinput0, vget_low_f32(vkernel3x), 0);
		#endif

		const float32x4_t vinput1 = vld1q_f32(input1); input1 += 4;
		#if defined(__aarch64__)
			voutput0 = vfmaq_lane_f32(voutput0, vinput1, vget_low_f32(vkernel0x), 1);
			voutput1 = vfmaq_lane_f32(voutput1, vinput1, vget_low_f32(vkernel1x), 1);
			voutput2 = vfmaq_lane_f32(voutput2, vinput1, vget_low_f32(vkernel2x), 1);
			voutput3 = vfmaq_lane_f32(voutput3, vinput1, vget_low_f32(vkernel3x), 1);
		#else
			voutput0 = vmlaq_lane_f32(voutput0, vinput1, vget_low_f32(vkernel0x), 1);
			voutput1 = vmlaq_lane_f32(voutput1, vinput1, vget_low_f32(vkernel1x), 1);
			voutput2 = vmlaq_lane_f32(voutput2, vinput1, vget_low_f32(vkernel2x), 1);
			voutput3 = vmlaq_lane_f32(voutput3, vinput1, vget_low_f32(vkernel3x), 1);
		#endif

		const float32x4_t vinput2 = vld1q_f32(input2); input2 += 4;
		#if defined(__aarch64__)
			voutput0 = vfmaq_lane_f32(voutput0, vinput2, vget_high_f32(vkernel0x), 0);
			voutput1 = vfmaq_lane_f32(voutput1, vinput2, vget_high_f32(vkernel1x), 0);
			voutput2 = vfmaq_lane_f32(voutput2, vinput2, vget_high_f32(vkernel2x), 0);
			voutput3 = vfmaq_lane_f32(voutput3, vinput2, vget_high_f32(vkernel3x), 0);
		#else
			voutput0 = vmlaq_lane_f32(voutput0, vinput2, vget_high_f32(vkernel0x), 0);
			voutput1 = vmlaq_lane_f32(voutput1, vinput2, vget_high_f32(vkernel1x), 0);
			voutput2 = vmlaq_lane_f32(voutput2, vinput2, vget_high_f32(vkernel2x), 0);
			voutput3 = vmlaq_lane_f32(voutput3, vinput2, vget_high_f32(vkernel3x), 0);
		#endif

		const float32x4_t vinput3 = vld1q_f32(input3); input3 += 4;
		#if defined(__aarch64__)
			voutput0 = vfmaq_lane_f32(voutput0, vinput3, vget_high_f32(vkernel0x), 1);
			voutput1 = vfmaq_lane_f32(voutput1, vinput3, vget_high_f32(vkernel1x), 1);
			voutput2 = vfmaq_lane_f32(voutput2, vinput3, vget_high_f32(vkernel2x), 1);
			voutput3 = vfmaq_lane_f32(voutput3, vinput3, vget_high_f32(vkernel3x), 1);
		#else
			voutput0 = vmlaq_lane_f32(voutput0, vinput3, vget_high_f32(vkernel0x), 1);
			voutput1 = vmlaq_lane_f32(voutput1, vinput3, vget_high_f32(vkernel1x), 1);
			voutput2 = vmlaq_lane_f32(voutput2, vinput3, vget_high_f32(vkernel2x), 1);
			voutput3 = vmlaq_lane_f32(voutput3, vinput3, vget_high_f32(vkernel3x), 1);
		#endif

		vst1q_f32(output0, voutput0); output0 += 4;
		vst1q_f32(output1, voutput1); output1 += 4;
		vst1q_f32(output2, voutput2); output2 += 4;
		vst1q_f32(output3, voutput3); output3 += 4;

		image_size -= 4;
	}
	if (image_size >= 2) {
		float32x2_t voutput0 = vld1_f32(output0);
		float32x2_t voutput1 = vld1_f32(output1);
		float32x2_t voutput2 = vld1_f32(output2);
		float32x2_t voutput3 = vld1_f32(output3);

		const float32x2_t vinput0 = vld1_f32(input0); input0 += 2;
		#if defined(__aarch64__)
			voutput0 = vfma_lane_f32(voutput0, vinput0, vget_low_f32(vkernel0x), 0);
			voutput1 = vfma_lane_f32(voutput1, vinput0, vget_low_f32(vkernel1x), 0);
			voutput2 = vfma_lane_f32(voutput2, vinput0, vget_low_f32(vkernel2x), 0);
			voutput3 = vfma_lane_f32(voutput3, vinput0, vget_low_f32(vkernel3x), 0);
		#else
			voutput0 = vmla_lane_f32(voutput0, vinput0, vget_low_f32(vkernel0x), 0);
			voutput1 = vmla_lane_f32(voutput1, vinput0, vget_low_f32(vkernel1x), 0);
			voutput2 = vmla_lane_f32(voutput2, vinput0, vget_low_f32(vkernel2x), 0);
			voutput3 = vmla_lane_f32(voutput3, vinput0, vget_low_f32(vkernel3x), 0);
		#endif

		const float32x2_t vinput1 = vld1_f32(input1); input1 += 2;
		#if defined(__aarch64__)
			voutput0 = vfma_lane_f32(voutput0, vinput1, vget_low_f32(vkernel0x), 1);
			voutput1 = vfma_lane_f32(voutput1, vinput1, vget_low_f32(vkernel1x), 1);
			voutput2 = vfma_lane_f32(voutput2, vinput1, vget_low_f32(vkernel2x), 1);
			voutput3 = vfma_lane_f32(voutput3, vinput1, vget_low_f32(vkernel3x), 1);
		#else
			voutput0 = vmla_lane_f32(voutput0, vinput1, vget_low_f32(vkernel0x), 1);
			voutput1 = vmla_lane_f32(voutput1, vinput1, vget_low_f32(vkernel1x), 1);
			voutput2 = vmla_lane_f32(voutput2, vinput1, vget_low_f32(vkernel2x), 1);
			voutput3 = vmla_lane_f32(voutput3, vinput1, vget_low_f32(vkernel3x), 1);
		#endif

		const float32x2_t vinput2 = vld1_f32(input2); input2 += 2;
		#if defined(__aarch64__)
			voutput0 = vfma_lane_f32(voutput0, vinput2, vget_high_f32(vkernel0x), 0);
			voutput1 = vfma_lane_f32(voutput1, vinput2, vget_high_f32(vkernel1x), 0);
			voutput2 = vfma_lane_f32(voutput2, vinput2, vget_high_f32(vkernel2x), 0);
			voutput3 = vfma_lane_f32(voutput3, vinput2, vget_high_f32(vkernel3x), 0);
		#else
			voutput0 = vmla_lane_f32(voutput0, vinput2, vget_high_f32(vkernel0x), 0);
			voutput1 = vmla_lane_f32(voutput1, vinput2, vget_high_f32(vkernel1x), 0);
			voutput2 = vmla_lane_f32(voutput2, vinput2, vget_high_f32(vkernel2x), 0);
			voutput3 = vmla_lane_f32(voutput3, vinput2, vget_high_f32(vkernel3x), 0);
		#endif

		const float32x2_t vinput3 = vld1_f32(input3); input3 += 2;
		#if defined(__aarch64__)
			voutput0 = vfma_lane_f32(voutput0, vinput3, vget_high_f32(vkernel0x), 1);
			voutput1 = vfma_lane_f32(voutput1, vinput3, vget_high_f32(vkernel1x), 1);
			voutput2 = vfma_lane_f32(voutput2, vinput3, vget_high_f32(vkernel2x), 1);
			voutput3 = vfma_lane_f32(voutput3, vinput3, vget_high_f32(vkernel3x), 1);
		#else
			voutput0 = vmla_lane_f32(voutput0, vinput3, vget_high_f32(vkernel0x), 1);
			voutput1 = vmla_lane_f32(voutput1, vinput3, vget_high_f32(vkernel1x), 1);
			voutput2 = vmla_lane_f32(voutput2, vinput3, vget_high_f32(vkernel2x), 1);
			voutput3 = vmla_lane_f32(voutput3, vinput3, vget_high_f32(vkernel3x), 1);
		#endif

		vst1_f32(output0, voutput0); output0 += 2;
		vst1_f32(output1, voutput1); output1 += 2;
		vst1_f32(output2, voutput2); output2 += 2;
		vst1_f32(output3, voutput3); output3 += 2;

		image_size -= 2;
	}
	if (image_size != 0) {
		float32x2_t voutput0 = vld1_dup_f32(output0);
		float32x2_t voutput1 = vld1_dup_f32(output1);
		float32x2_t voutput2 = vld1_dup_f32(output2);
		float32x2_t voutput3 = vld1_dup_f32(output3);

		const float32x2_t vinput0 = vld1_dup_f32(input0);
		#if defined(__aarch64__)
			voutput0 = vfma_lane_f32(voutput0, vinput0, vget_low_f32(vkernel0x), 0);
			voutput1 = vfma_lane_f32(voutput1, vinput0, vget_low_f32(vkernel1x), 0);
			voutput2 = vfma_lane_f32(voutput2, vinput0, vget_low_f32(vkernel2x), 0);
			voutput3 = vfma_lane_f32(voutput3, vinput0, vget_low_f32(vkernel3x), 0);
		#else
			voutput0 = vmla_lane_f32(voutput0, vinput0, vget_low_f32(vkernel0x), 0);
			voutput1 = vmla_lane_f32(voutput1, vinput0, vget_low_f32(vkernel1x), 0);
			voutput2 = vmla_lane_f32(voutput2, vinput0, vget_low_f32(vkernel2x), 0);
			voutput3 = vmla_lane_f32(voutput3, vinput0, vget_low_f32(vkernel3x), 0);
		#endif

		const float32x2_t vinput1 = vld1_dup_f32(input1);
		#if defined(__aarch64__)
			voutput0 = vfma_lane_f32(voutput0, vinput1, vget_low_f32(vkernel0x), 1);
			voutput1 = vfma_lane_f32(voutput1, vinput1, vget_low_f32(vkernel1x), 1);
			voutput2 = vfma_lane_f32(voutput2, vinput1, vget_low_f32(vkernel2x), 1);
			voutput3 = vfma_lane_f32(voutput3, vinput1, vget_low_f32(vkernel3x), 1);
		#else
			voutput0 = vmla_lane_f32(voutput0, vinput1, vget_low_f32(vkernel0x), 1);
			voutput1 = vmla_lane_f32(voutput1, vinput1, vget_low_f32(vkernel1x), 1);
			voutput2 = vmla_lane_f32(voutput2, vinput1, vget_low_f32(vkernel2x), 1);
			voutput3 = vmla_lane_f32(voutput3, vinput1, vget_low_f32(vkernel3x), 1);
		#endif

		const float32x2_t vinput2 = vld1_dup_f32(input2);
		#if defined(__aarch64__)
			voutput0 = vfma_lane_f32(voutput0, vinput2, vget_high_f32(vkernel0x), 0);
			voutput1 = vfma_lane_f32(voutput1, vinput2, vget_high_f32(vkernel1x), 0);
			voutput2 = vfma_lane_f32(voutput2, vinput2, vget_high_f32(vkernel2x), 0);
			voutput3 = vfma_lane_f32(voutput3, vinput2, vget_high_f32(vkernel3x), 0);
		#else
			voutput0 = vmla_lane_f32(voutput0, vinput2, vget_high_f32(vkernel0x), 0);
			voutput1 = vmla_lane_f32(voutput1, vinput2, vget_high_f32(vkernel1x), 0);
			voutput2 = vmla_lane_f32(voutput2, vinput2, vget_high_f32(vkernel2x), 0);
			voutput3 = vmla_lane_f32(voutput3, vinput2, vget_high_f32(vkernel3x), 0);
		#endif

		const float32x2_t vinput3 = vld1_dup_f32(input3);
		#if defined(__aarch64__)
			voutput0 = vfma_lane_f32(voutput0, vinput3, vget_high_f32(vkernel0x), 1);
			voutput1 = vfma_lane_f32(voutput1, vinput3, vget_high_f32(vkernel1x), 1);
			voutput2 = vfma_lane_f32(voutput2, vinput3, vget_high_f32(vkernel2x), 1);
			voutput3 = vfma_lane_f32(voutput3, vinput3, vget_high_f32(vkernel3x), 1);
		#else
			voutput0 = vmla_lane_f32(voutput0, vinput3, vget_high_f32(vkernel0x), 1);
			voutput1 = vmla_lane_f32(voutput1, vinput3, vget_high_f32(vkernel1x), 1);
			voutput2 = vmla_lane_f32(voutput2, vinput3, vget_high_f32(vkernel2x), 1);
			voutput3 = vmla_lane_f32(voutput3, vinput3, vget_high_f32(vkernel3x), 1);
		#endif

		vst1_lane_f32(output0, voutput0, 0);
		vst1_lane_f32(output1, voutput1, 0);
		vst1_lane_f32(output2, voutput2, 0);
		vst1_lane_f32(output3, voutput3, 0);
	}
}

void nnp_conv1x1_upto_4x4__neon(
	uint32_t input_channels_subblock_size,
	uint32_t output_channels_subblock_size,
	size_t input_channels,
	size_t image_size,
	const float* input,
	const float* kernel,
	float* output)
{
	const float*restrict input0 = input;
	const float*restrict input1 = input_channels_subblock_size > 1 ? input0 + image_size : input0;
	const float*restrict input2 = input_channels_subblock_size > 2 ? input1 + image_size : input1;
	const float*restrict input3 = input_channels_subblock_size > 3 ? input2 + image_size : input2;

	const float*restrict kernel0 = kernel;
	const float*restrict kernel1 = output_channels_subblock_size > 1 ? kernel0 + input_channels : kernel0;
	const float*restrict kernel2 = output_channels_subblock_size > 2 ? kernel1 + input_channels : kernel1;
	const float*restrict kernel3 = output_channels_subblock_size > 3 ? kernel2 + input_channels : kernel2;

	float32x4_t vkernel0x = vld1q_dup_f32(kernel0);
	float32x4_t vkernel1x = vld1q_dup_f32(kernel1);
	float32x4_t vkernel2x = vld1q_dup_f32(kernel2);
	float32x4_t vkernel3x = vld1q_dup_f32(kernel3);
	if (input_channels_subblock_size > 1) {
		vkernel0x = vld1q_lane_f32(kernel0 + 1, vkernel0x, 1);
		vkernel1x = vld1q_lane_f32(kernel1 + 1, vkernel1x, 1);
		vkernel2x = vld1q_lane_f32(kernel2 + 1, vkernel2x, 1);
		vkernel3x = vld1q_lane_f32(kernel3 + 1, vkernel3x, 1);
		if (input_channels_subblock_size > 2) {
			vkernel0x = vld1q_lane_f32(kernel0 + 2, vkernel0x, 2);
			vkernel1x = vld1q_lane_f32(kernel1 + 2, vkernel1x, 2);
			vkernel2x = vld1q_lane_f32(kernel2 + 2, vkernel2x, 2);
			vkernel3x = vld1q_lane_f32(kernel3 + 2, vkernel3x, 2);
			if (input_channels_subblock_size > 3) {
				vkernel0x = vld1q_lane_f32(kernel0 + 3, vkernel0x, 3);
				vkernel1x = vld1q_lane_f32(kernel1 + 3, vkernel1x, 3);
				vkernel2x = vld1q_lane_f32(kernel2 + 3, vkernel2x, 3);
				vkernel3x = vld1q_lane_f32(kernel3 + 3, vkernel3x, 3);
			}
		}
	}

	float*restrict output0 = output;
	float*restrict output1 = output_channels_subblock_size > 1 ? output0 + image_size : output0;
	float*restrict output2 = output_channels_subblock_size > 2 ? output1 + image_size : output1;
	float*restrict output3 = output_channels_subblock_size > 3 ? output2 + image_size : output3;
	while (image_size >= 4) {
		float32x4_t voutput0 = vld1q_f32(output0);
		float32x4_t voutput1 = vld1q_f32(output1);
		float32x4_t voutput2 = vld1q_f32(output2);
		float32x4_t voutput3 = vld1q_f32(output3);<commit_msg>Fix bug in direct 1x1 NEON ukernel<commit_after>#include <stddef.h>
#include <stdint.h>

#include <nnpack/macros.h>
#include <nnpack/arm_neon.h>


void nnp_conv1x1_only_4x4__neon(
	size_t input_channels,
	size_t image_size,
	const float* input,
	const float* kernel,
	float* output)
{
	const float* input0 = input;
	const float* input1 = input0 + image_size;
	const float* input2 = input1 + image_size;
	const float* input3 = input2 + image_size;

	const float32x4_t vkernel0x = vld1q_f32(kernel);
	kernel += input_channels;
	const float32x4_t vkernel1x = vld1q_f32(kernel);
	kernel += input_channels;
	const float32x4_t vkernel2x = vld1q_f32(kernel);
	kernel += input_channels;
	const float32x4_t vkernel3x = vld1q_f32(kernel);

	float* output0 = output;
	float* output1 = output0 + image_size;
	float* output2 = output1 + image_size;
	float* output3 = output2 + image_size;
	while (image_size >= 4) {
		float32x4_t voutput0 = vld1q_f32(output0);
		float32x4_t voutput1 = vld1q_f32(output1);
		float32x4_t voutput2 = vld1q_f32(output2);
		float32x4_t voutput3 = vld1q_f32(output3);

		const float32x4_t vinput0 = vld1q_f32(input0); input0 += 4;
		#if defined(__aarch64__)
			voutput0 = vfmaq_lane_f32(voutput0, vinput0, vget_low_f32(vkernel0x), 0);
			voutput1 = vfmaq_lane_f32(voutput1, vinput0, vget_low_f32(vkernel1x), 0);
			voutput2 = vfmaq_lane_f32(voutput2, vinput0, vget_low_f32(vkernel2x), 0);
			voutput3 = vfmaq_lane_f32(voutput3, vinput0, vget_low_f32(vkernel3x), 0);
		#else
			voutput0 = vmlaq_lane_f32(voutput0, vinput0, vget_low_f32(vkernel0x), 0);
			voutput1 = vmlaq_lane_f32(voutput1, vinput0, vget_low_f32(vkernel1x), 0);
			voutput2 = vmlaq_lane_f32(voutput2, vinput0, vget_low_f32(vkernel2x), 0);
			voutput3 = vmlaq_lane_f32(voutput3, vinput0, vget_low_f32(vkernel3x), 0);
		#endif

		const float32x4_t vinput1 = vld1q_f32(input1); input1 += 4;
		#if defined(__aarch64__)
			voutput0 = vfmaq_lane_f32(voutput0, vinput1, vget_low_f32(vkernel0x), 1);
			voutput1 = vfmaq_lane_f32(voutput1, vinput1, vget_low_f32(vkernel1x), 1);
			voutput2 = vfmaq_lane_f32(voutput2, vinput1, vget_low_f32(vkernel2x), 1);
			voutput3 = vfmaq_lane_f32(voutput3, vinput1, vget_low_f32(vkernel3x), 1);
		#else
			voutput0 = vmlaq_lane_f32(voutput0, vinput1, vget_low_f32(vkernel0x), 1);
			voutput1 = vmlaq_lane_f32(voutput1, vinput1, vget_low_f32(vkernel1x), 1);
			voutput2 = vmlaq_lane_f32(voutput2, vinput1, vget_low_f32(vkernel2x), 1);
			voutput3 = vmlaq_lane_f32(voutput3, vinput1, vget_low_f32(vkernel3x), 1);
		#endif

		const float32x4_t vinput2 = vld1q_f32(input2); input2 += 4;
		#if defined(__aarch64__)
			voutput0 = vfmaq_lane_f32(voutput0, vinput2, vget_high_f32(vkernel0x), 0);
			voutput1 = vfmaq_lane_f32(voutput1, vinput2, vget_high_f32(vkernel1x), 0);
			voutput2 = vfmaq_lane_f32(voutput2, vinput2, vget_high_f32(vkernel2x), 0);
			voutput3 = vfmaq_lane_f32(voutput3, vinput2, vget_high_f32(vkernel3x), 0);
		#else
			voutput0 = vmlaq_lane_f32(voutput0, vinput2, vget_high_f32(vkernel0x), 0);
			voutput1 = vmlaq_lane_f32(voutput1, vinput2, vget_high_f32(vkernel1x), 0);
			voutput2 = vmlaq_lane_f32(voutput2, vinput2, vget_high_f32(vkernel2x), 0);
			voutput3 = vmlaq_lane_f32(voutput3, vinput2, vget_high_f32(vkernel3x), 0);
		#endif

		const float32x4_t vinput3 = vld1q_f32(input3); input3 += 4;
		#if defined(__aarch64__)
			voutput0 = vfmaq_lane_f32(voutput0, vinput3, vget_high_f32(vkernel0x), 1);
			voutput1 = vfmaq_lane_f32(voutput1, vinput3, vget_high_f32(vkernel1x), 1);
			voutput2 = vfmaq_lane_f32(voutput2, vinput3, vget_high_f32(vkernel2x), 1);
			voutput3 = vfmaq_lane_f32(voutput3, vinput3, vget_high_f32(vkernel3x), 1);
		#else
			voutput0 = vmlaq_lane_f32(voutput0, vinput3, vget_high_f32(vkernel0x), 1);
			voutput1 = vmlaq_lane_f32(voutput1, vinput3, vget_high_f32(vkernel1x), 1);
			voutput2 = vmlaq_lane_f32(voutput2, vinput3, vget_high_f32(vkernel2x), 1);
			voutput3 = vmlaq_lane_f32(voutput3, vinput3, vget_high_f32(vkernel3x), 1);
		#endif

		vst1q_f32(output0, voutput0); output0 += 4;
		vst1q_f32(output1, voutput1); output1 += 4;
		vst1q_f32(output2, voutput2); output2 += 4;
		vst1q_f32(output3, voutput3); output3 += 4;

		image_size -= 4;
	}
	if (image_size >= 2) {
		float32x2_t voutput0 = vld1_f32(output0);
		float32x2_t voutput1 = vld1_f32(output1);
		float32x2_t voutput2 = vld1_f32(output2);
		float32x2_t voutput3 = vld1_f32(output3);

		const float32x2_t vinput0 = vld1_f32(input0); input0 += 2;
		#if defined(__aarch64__)
			voutput0 = vfma_lane_f32(voutput0, vinput0, vget_low_f32(vkernel0x), 0);
			voutput1 = vfma_lane_f32(voutput1, vinput0, vget_low_f32(vkernel1x), 0);
			voutput2 = vfma_lane_f32(voutput2, vinput0, vget_low_f32(vkernel2x), 0);
			voutput3 = vfma_lane_f32(voutput3, vinput0, vget_low_f32(vkernel3x), 0);
		#else
			voutput0 = vmla_lane_f32(voutput0, vinput0, vget_low_f32(vkernel0x), 0);
			voutput1 = vmla_lane_f32(voutput1, vinput0, vget_low_f32(vkernel1x), 0);
			voutput2 = vmla_lane_f32(voutput2, vinput0, vget_low_f32(vkernel2x), 0);
			voutput3 = vmla_lane_f32(voutput3, vinput0, vget_low_f32(vkernel3x), 0);
		#endif

		const float32x2_t vinput1 = vld1_f32(input1); input1 += 2;
		#if defined(__aarch64__)
			voutput0 = vfma_lane_f32(voutput0, vinput1, vget_low_f32(vkernel0x), 1);
			voutput1 = vfma_lane_f32(voutput1, vinput1, vget_low_f32(vkernel1x), 1);
			voutput2 = vfma_lane_f32(voutput2, vinput1, vget_low_f32(vkernel2x), 1);
			voutput3 = vfma_lane_f32(voutput3, vinput1, vget_low_f32(vkernel3x), 1);
		#else
			voutput0 = vmla_lane_f32(voutput0, vinput1, vget_low_f32(vkernel0x), 1);
			voutput1 = vmla_lane_f32(voutput1, vinput1, vget_low_f32(vkernel1x), 1);
			voutput2 = vmla_lane_f32(voutput2, vinput1, vget_low_f32(vkernel2x), 1);
			voutput3 = vmla_lane_f32(voutput3, vinput1, vget_low_f32(vkernel3x), 1);
		#endif

		const float32x2_t vinput2 = vld1_f32(input2); input2 += 2;
		#if defined(__aarch64__)
			voutput0 = vfma_lane_f32(voutput0, vinput2, vget_high_f32(vkernel0x), 0);
			voutput1 = vfma_lane_f32(voutput1, vinput2, vget_high_f32(vkernel1x), 0);
			voutput2 = vfma_lane_f32(voutput2, vinput2, vget_high_f32(vkernel2x), 0);
			voutput3 = vfma_lane_f32(voutput3, vinput2, vget_high_f32(vkernel3x), 0);
		#else
			voutput0 = vmla_lane_f32(voutput0, vinput2, vget_high_f32(vkernel0x), 0);
			voutput1 = vmla_lane_f32(voutput1, vinput2, vget_high_f32(vkernel1x), 0);
			voutput2 = vmla_lane_f32(voutput2, vinput2, vget_high_f32(vkernel2x), 0);
			voutput3 = vmla_lane_f32(voutput3, vinput2, vget_high_f32(vkernel3x), 0);
		#endif

		const float32x2_t vinput3 = vld1_f32(input3); input3 += 2;
		#if defined(__aarch64__)
			voutput0 = vfma_lane_f32(voutput0, vinput3, vget_high_f32(vkernel0x), 1);
			voutput1 = vfma_lane_f32(voutput1, vinput3, vget_high_f32(vkernel1x), 1);
			voutput2 = vfma_lane_f32(voutput2, vinput3, vget_high_f32(vkernel2x), 1);
			voutput3 = vfma_lane_f32(voutput3, vinput3, vget_high_f32(vkernel3x), 1);
		#else
			voutput0 = vmla_lane_f32(voutput0, vinput3, vget_high_f32(vkernel0x), 1);
			voutput1 = vmla_lane_f32(voutput1, vinput3, vget_high_f32(vkernel1x), 1);
			voutput2 = vmla_lane_f32(voutput2, vinput3, vget_high_f32(vkernel2x), 1);
			voutput3 = vmla_lane_f32(voutput3, vinput3, vget_high_f32(vkernel3x), 1);
		#endif

		vst1_f32(output0, voutput0); output0 += 2;
		vst1_f32(output1, voutput1); output1 += 2;
		vst1_f32(output2, voutput2); output2 += 2;
		vst1_f32(output3, voutput3); output3 += 2;

		image_size -= 2;
	}
	if (image_size != 0) {
		float32x2_t voutput0 = vld1_dup_f32(output0);
		float32x2_t voutput1 = vld1_dup_f32(output1);
		float32x2_t voutput2 = vld1_dup_f32(output2);
		float32x2_t voutput3 = vld1_dup_f32(output3);

		const float32x2_t vinput0 = vld1_dup_f32(input0);
		#if defined(__aarch64__)
			voutput0 = vfma_lane_f32(voutput0, vinput0, vget_low_f32(vkernel0x), 0);
			voutput1 = vfma_lane_f32(voutput1, vinput0, vget_low_f32(vkernel1x), 0);
			voutput2 = vfma_lane_f32(voutput2, vinput0, vget_low_f32(vkernel2x), 0);
			voutput3 = vfma_lane_f32(voutput3, vinput0, vget_low_f32(vkernel3x), 0);
		#else
			voutput0 = vmla_lane_f32(voutput0, vinput0, vget_low_f32(vkernel0x), 0);
			voutput1 = vmla_lane_f32(voutput1, vinput0, vget_low_f32(vkernel1x), 0);
			voutput2 = vmla_lane_f32(voutput2, vinput0, vget_low_f32(vkernel2x), 0);
			voutput3 = vmla_lane_f32(voutput3, vinput0, vget_low_f32(vkernel3x), 0);
		#endif

		const float32x2_t vinput1 = vld1_dup_f32(input1);
		#if defined(__aarch64__)
			voutput0 = vfma_lane_f32(voutput0, vinput1, vget_low_f32(vkernel0x), 1);
			voutput1 = vfma_lane_f32(voutput1, vinput1, vget_low_f32(vkernel1x), 1);
			voutput2 = vfma_lane_f32(voutput2, vinput1, vget_low_f32(vkernel2x), 1);
			voutput3 = vfma_lane_f32(voutput3, vinput1, vget_low_f32(vkernel3x), 1);
		#else
			voutput0 = vmla_lane_f32(voutput0, vinput1, vget_low_f32(vkernel0x), 1);
			voutput1 = vmla_lane_f32(voutput1, vinput1, vget_low_f32(vkernel1x), 1);
			voutput2 = vmla_lane_f32(voutput2, vinput1, vget_low_f32(vkernel2x), 1);
			voutput3 = vmla_lane_f32(voutput3, vinput1, vget_low_f32(vkernel3x), 1);
		#endif

		const float32x2_t vinput2 = vld1_dup_f32(input2);
		#if defined(__aarch64__)
			voutput0 = vfma_lane_f32(voutput0, vinput2, vget_high_f32(vkernel0x), 0);
			voutput1 = vfma_lane_f32(voutput1, vinput2, vget_high_f32(vkernel1x), 0);
			voutput2 = vfma_lane_f32(voutput2, vinput2, vget_high_f32(vkernel2x), 0);
			voutput3 = vfma_lane_f32(voutput3, vinput2, vget_high_f32(vkernel3x), 0);
		#else
			voutput0 = vmla_lane_f32(voutput0, vinput2, vget_high_f32(vkernel0x), 0);
			voutput1 = vmla_lane_f32(voutput1, vinput2, vget_high_f32(vkernel1x), 0);
			voutput2 = vmla_lane_f32(voutput2, vinput2, vget_high_f32(vkernel2x), 0);
			voutput3 = vmla_lane_f32(voutput3, vinput2, vget_high_f32(vkernel3x), 0);
		#endif

		const float32x2_t vinput3 = vld1_dup_f32(input3);
		#if defined(__aarch64__)
			voutput0 = vfma_lane_f32(voutput0, vinput3, vget_high_f32(vkernel0x), 1);
			voutput1 = vfma_lane_f32(voutput1, vinput3, vget_high_f32(vkernel1x), 1);
			voutput2 = vfma_lane_f32(voutput2, vinput3, vget_high_f32(vkernel2x), 1);
			voutput3 = vfma_lane_f32(voutput3, vinput3, vget_high_f32(vkernel3x), 1);
		#else
			voutput0 = vmla_lane_f32(voutput0, vinput3, vget_high_f32(vkernel0x), 1);
			voutput1 = vmla_lane_f32(voutput1, vinput3, vget_high_f32(vkernel1x), 1);
			voutput2 = vmla_lane_f32(voutput2, vinput3, vget_high_f32(vkernel2x), 1);
			voutput3 = vmla_lane_f32(voutput3, vinput3, vget_high_f32(vkernel3x), 1);
		#endif

		vst1_lane_f32(output0, voutput0, 0);
		vst1_lane_f32(output1, voutput1, 0);
		vst1_lane_f32(output2, voutput2, 0);
		vst1_lane_f32(output3, voutput3, 0);
	}
}

void nnp_conv1x1_upto_4x4__neon(
	uint32_t input_channels_subblock_size,
	uint32_t output_channels_subblock_size,
	size_t input_channels,
	size_t image_size,
	const float* input,
	const float* kernel,
	float* output)
{
	const float*restrict input0 = input;
	const float*restrict input1 = input_channels_subblock_size > 1 ? input0 + image_size : input0;
	const float*restrict input2 = input_channels_subblock_size > 2 ? input1 + image_size : input1;
	const float*restrict input3 = input_channels_subblock_size > 3 ? input2 + image_size : input2;

	const float*restrict kernel0 = kernel;
	const float*restrict kernel1 = output_channels_subblock_size > 1 ? kernel0 + input_channels : kernel0;
	const float*restrict kernel2 = output_channels_subblock_size > 2 ? kernel1 + input_channels : kernel1;
	const float*restrict kernel3 = output_channels_subblock_size > 3 ? kernel2 + input_channels : kernel2;

	float32x4_t vkernel0x = vld1q_dup_f32(kernel0);
	float32x4_t vkernel1x = vld1q_dup_f32(kernel1);
	float32x4_t vkernel2x = vld1q_dup_f32(kernel2);
	float32x4_t vkernel3x = vld1q_dup_f32(kernel3);
	if (input_channels_subblock_size > 1) {
		vkernel0x = vld1q_lane_f32(kernel0 + 1, vkernel0x, 1);
		vkernel1x = vld1q_lane_f32(kernel1 + 1, vkernel1x, 1);
		vkernel2x = vld1q_lane_f32(kernel2 + 1, vkernel2x, 1);
		vkernel3x = vld1q_lane_f32(kernel3 + 1, vkernel3x, 1);
		if (input_channels_subblock_size > 2) {
			vkernel0x = vld1q_lane_f32(kernel0 + 2, vkernel0x, 2);
			vkernel1x = vld1q_lane_f32(kernel1 + 2, vkernel1x, 2);
			vkernel2x = vld1q_lane_f32(kernel2 + 2, vkernel2x, 2);
			vkernel3x = vld1q_lane_f32(kernel3 + 2, vkernel3x, 2);
			if (input_channels_subblock_size > 3) {
				vkernel0x = vld1q_lane_f32(kernel0 + 3, vkernel0x, 3);
				vkernel1x = vld1q_lane_f32(kernel1 + 3, vkernel1x, 3);
				vkernel2x = vld1q_lane_f32(kernel2 + 3, vkernel2x, 3);
				vkernel3x = vld1q_lane_f32(kernel3 + 3, vkernel3x, 3);
			}
		}
	}

	float*restrict output0 = output;
	float*restrict output1 = output_channels_subblock_size > 1 ? output0 + image_size : output0;
	float*restrict output2 = output_channels_subblock_size > 2 ? output1 + image_size : output1;
	float*restrict output3 = output_channels_subblock_size > 3 ? output2 + image_size : output2;
	while (image_size >= 4) {
		float32x4_t voutput0 = vld1q_f32(output0);
		float32x4_t voutput1 = vld1q_f32(output1);
		float32x4_t voutput2 = vld1q_f32(output2);
		float32x4_t voutput3 = vld1q_f32(output3);
{'fraction_non_alphanumeric': 0.11049765830255492, 'fraction_numerical': 0.10958382515325743, 'mean_word_length': 6.305702364394993, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3793043', 'n_tokens_mistral': 13906, 'n_tokens_neox': 12306, 'n_words': 1919}