<commit_before>/*
 *  Copyright (c) 2014 The WebM project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include <arm_neon.h>

#include "./vpx_dsp_rtcd.h"
#include "./vpx_config.h"

#include "vpx/vpx_integer.h"
#include "vpx_dsp/arm/mem_neon.h"
#include "vpx_ports/mem.h"

static INLINE int horizontal_add_s16x8(const int16x8_t v_16x8) {
  const int32x4_t a = vpaddlq_s16(v_16x8);
  const int64x2_t b = vpaddlq_s32(a);
  const int32x2_t c = vadd_s32(vreinterpret_s32_s64(vget_low_s64(b)),
                               vreinterpret_s32_s64(vget_high_s64(b)));
  return vget_lane_s32(c, 0);
}

static INLINE int horizontal_add_s32x4(const int32x4_t v_32x4) {
  const int64x2_t b = vpaddlq_s32(v_32x4);
  const int32x2_t c = vadd_s32(vreinterpret_s32_s64(vget_low_s64(b)),
                               vreinterpret_s32_s64(vget_high_s64(b)));
  return vget_lane_s32(c, 0);
}

// w * h must be less than 2048 or sum_s16 may overflow.
// Process a block of width 4 four rows at a time.
static void variance_neon_w4x4(const uint8_t *a, int a_stride, const uint8_t *b,
                               int b_stride, int h, uint32_t *sse, int *sum) {
  int i;
  int16x8_t sum_s16 = vdupq_n_s16(0);
  int32x4_t sse_lo_s32 = vdupq_n_s32(0);
  int32x4_t sse_hi_s32 = vdupq_n_s32(0);

  for (i = 0; i < h; i += 4) {
    const uint8x16_t a_u8 = load_unaligned_u8q(a, a_stride);
    const uint8x16_t b_u8 = load_unaligned_u8q(b, b_stride);
    const uint16x8_t diff_lo_u16 =
        vsubl_u8(vget_low_u8(a_u8), vget_low_u8(b_u8));
    const uint16x8_t diff_hi_u16 =
        vsubl_u8(vget_high_u8(a_u8), vget_high_u8(b_u8));

    const int16x8_t diff_lo_s16 = vreinterpretq_s16_u16(diff_lo_u16);
    const int16x8_t diff_hi_s16 = vreinterpretq_s16_u16(diff_hi_u16);

    sum_s16 = vaddq_s16(sum_s16, diff_lo_s16);
    sum_s16 = vaddq_s16(sum_s16, diff_hi_s16);

    sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_low_s16(diff_lo_s16),
                           vget_low_s16(diff_lo_s16));
    sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_high_s16(diff_lo_s16),
                           vget_high_s16(diff_lo_s16));

    sse_hi_s32 = vmlal_s16(sse_hi_s32, vget_low_s16(diff_hi_s16),
                           vget_low_s16(diff_hi_s16));
    sse_hi_s32 = vmlal_s16(sse_hi_s32, vget_high_s16(diff_hi_s16),
                           vget_high_s16(diff_hi_s16));

    a += 4 * a_stride;
    b += 4 * b_stride;
  }

  *sum = horizontal_add_s16x8(sum_s16);
  *sse = (uint32_t)horizontal_add_s32x4(vaddq_s32(sse_lo_s32, sse_hi_s32));
}

// w * h must be less than 2048 or sum_s16 may overflow.
// Process a block of any size where the width is divisible by 16.
static void variance_neon_w16(const uint8_t *a, int a_stride, const uint8_t *b,
                              int b_stride, int w, int h, uint32_t *sse,
                              int *sum) {
  int i, j;
  int16x8_t sum_s16 = vdupq_n_s16(0);
  int32x4_t sse_lo_s32 = vdupq_n_s32(0);
  int32x4_t sse_hi_s32 = vdupq_n_s32(0);

  for (i = 0; i < h; ++i) {
    for (j = 0; j < w; j += 16) {
      const uint8x16_t a_u8 = vld1q_u8(a + j);
      const uint8x16_t b_u8 = vld1q_u8(b + j);

      const uint16x8_t diff_lo_u16 =
          vsubl_u8(vget_low_u8(a_u8), vget_low_u8(b_u8));
      const uint16x8_t diff_hi_u16 =
          vsubl_u8(vget_high_u8(a_u8), vget_high_u8(b_u8));

      const int16x8_t diff_lo_s16 = vreinterpretq_s16_u16(diff_lo_u16);
      const int16x8_t diff_hi_s16 = vreinterpretq_s16_u16(diff_hi_u16);

      sum_s16 = vaddq_s16(sum_s16, diff_lo_s16);
      sum_s16 = vaddq_s16(sum_s16, diff_hi_s16);

      sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_low_s16(diff_lo_s16),
                             vget_low_s16(diff_lo_s16));
      sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_high_s16(diff_lo_s16),
                             vget_high_s16(diff_lo_s16));

      sse_hi_s32 = vmlal_s16(sse_hi_s32, vget_low_s16(diff_hi_s16),
                             vget_low_s16(diff_hi_s16));
      sse_hi_s32 = vmlal_s16(sse_hi_s32, vget_high_s16(diff_hi_s16),
                             vget_high_s16(diff_hi_s16));
    }
    a += a_stride;
    b += b_stride;
  }

  *sum = horizontal_add_s16x8(sum_s16);
  *sse = (unsigned int)horizontal_add_s32x4(vaddq_s32(sse_lo_s32, sse_hi_s32));
}

// w * h must be less than 2048 or sum_s16 may overflow.
// Process a block of width 8 two rows at a time.
static void variance_neon_w8x2(const uint8_t *a, int a_stride, const uint8_t *b,
                               int b_stride, int h, uint32_t *sse, int *sum) {
  int i = 0;
  int16x8_t sum_s16 = vdupq_n_s16(0);
  int32x4_t sse_lo_s32 = vdupq_n_s32(0);
  int32x4_t sse_hi_s32 = vdupq_n_s32(0);

  do {
    const uint8x8_t a_0_u8 = vld1_u8(a);
    const uint8x8_t a_1_u8 = vld1_u8(a + a_stride);
    const uint8x8_t b_0_u8 = vld1_u8(b);
    const uint8x8_t b_1_u8 = vld1_u8(b + b_stride);
    const uint16x8_t diff_0_u16 = vsubl_u8(a_0_u8, b_0_u8);
    const uint16x8_t diff_1_u16 = vsubl_u8(a_1_u8, b_1_u8);
    const int16x8_t diff_0_s16 = vreinterpretq_s16_u16(diff_0_u16);
    const int16x8_t diff_1_s16 = vreinterpretq_s16_u16(diff_1_u16);
    sum_s16 = vaddq_s16(sum_s16, diff_0_s16);
    sum_s16 = vaddq_s16(sum_s16, diff_1_s16);
    sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_low_s16(diff_0_s16),
                           vget_low_s16(diff_0_s16));
    sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_low_s16(diff_1_s16),<commit_msg>variance neon: assert overflow conditions<commit_after>/*
 *  Copyright (c) 2014 The WebM project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include <arm_neon.h>
#include <assert.h>

#include "./vpx_dsp_rtcd.h"
#include "./vpx_config.h"

#include "vpx/vpx_integer.h"
#include "vpx_dsp/arm/mem_neon.h"
#include "vpx_ports/mem.h"

static INLINE int horizontal_add_s16x8(const int16x8_t v_16x8) {
  const int32x4_t a = vpaddlq_s16(v_16x8);
  const int64x2_t b = vpaddlq_s32(a);
  const int32x2_t c = vadd_s32(vreinterpret_s32_s64(vget_low_s64(b)),
                               vreinterpret_s32_s64(vget_high_s64(b)));
  return vget_lane_s32(c, 0);
}

static INLINE int horizontal_add_s32x4(const int32x4_t v_32x4) {
  const int64x2_t b = vpaddlq_s32(v_32x4);
  const int32x2_t c = vadd_s32(vreinterpret_s32_s64(vget_low_s64(b)),
                               vreinterpret_s32_s64(vget_high_s64(b)));
  return vget_lane_s32(c, 0);
}

// The variance helper functions use int16_t for sum. 8 values are accumulated
// and then added (at which point they expand up to int32_t). To avoid overflow,
// there can be no more than 32767 / 255 ~= 128 values accumulated in each
// column. For a 32x32 buffer, this results in 32 / 8 = 4 values per row * 32
// rows = 128. Asserts have been added to each function to warn against reaching
// this limit.

// Process a block of width 4 four rows at a time.
static void variance_neon_w4x4(const uint8_t *a, int a_stride, const uint8_t *b,
                               int b_stride, int h, uint32_t *sse, int *sum) {
  int i;
  int16x8_t sum_s16 = vdupq_n_s16(0);
  int32x4_t sse_lo_s32 = vdupq_n_s32(0);
  int32x4_t sse_hi_s32 = vdupq_n_s32(0);

  // Since width is only 4, sum_s16 only loads a half row per loop.
  assert(h <= 256);

  for (i = 0; i < h; i += 4) {
    const uint8x16_t a_u8 = load_unaligned_u8q(a, a_stride);
    const uint8x16_t b_u8 = load_unaligned_u8q(b, b_stride);
    const uint16x8_t diff_lo_u16 =
        vsubl_u8(vget_low_u8(a_u8), vget_low_u8(b_u8));
    const uint16x8_t diff_hi_u16 =
        vsubl_u8(vget_high_u8(a_u8), vget_high_u8(b_u8));

    const int16x8_t diff_lo_s16 = vreinterpretq_s16_u16(diff_lo_u16);
    const int16x8_t diff_hi_s16 = vreinterpretq_s16_u16(diff_hi_u16);

    sum_s16 = vaddq_s16(sum_s16, diff_lo_s16);
    sum_s16 = vaddq_s16(sum_s16, diff_hi_s16);

    sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_low_s16(diff_lo_s16),
                           vget_low_s16(diff_lo_s16));
    sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_high_s16(diff_lo_s16),
                           vget_high_s16(diff_lo_s16));

    sse_hi_s32 = vmlal_s16(sse_hi_s32, vget_low_s16(diff_hi_s16),
                           vget_low_s16(diff_hi_s16));
    sse_hi_s32 = vmlal_s16(sse_hi_s32, vget_high_s16(diff_hi_s16),
                           vget_high_s16(diff_hi_s16));

    a += 4 * a_stride;
    b += 4 * b_stride;
  }

  *sum = horizontal_add_s16x8(sum_s16);
  *sse = (uint32_t)horizontal_add_s32x4(vaddq_s32(sse_lo_s32, sse_hi_s32));
}

// Process a block of any size where the width is divisible by 16.
static void variance_neon_w16(const uint8_t *a, int a_stride, const uint8_t *b,
                              int b_stride, int w, int h, uint32_t *sse,
                              int *sum) {
  int i, j;
  int16x8_t sum_s16 = vdupq_n_s16(0);
  int32x4_t sse_lo_s32 = vdupq_n_s32(0);
  int32x4_t sse_hi_s32 = vdupq_n_s32(0);

  // The loop loads 16 values at a time but doubles them up when accumulating
  // into sum_s16.
  assert(w / 8 * h <= 128);

  for (i = 0; i < h; ++i) {
    for (j = 0; j < w; j += 16) {
      const uint8x16_t a_u8 = vld1q_u8(a + j);
      const uint8x16_t b_u8 = vld1q_u8(b + j);

      const uint16x8_t diff_lo_u16 =
          vsubl_u8(vget_low_u8(a_u8), vget_low_u8(b_u8));
      const uint16x8_t diff_hi_u16 =
          vsubl_u8(vget_high_u8(a_u8), vget_high_u8(b_u8));

      const int16x8_t diff_lo_s16 = vreinterpretq_s16_u16(diff_lo_u16);
      const int16x8_t diff_hi_s16 = vreinterpretq_s16_u16(diff_hi_u16);

      sum_s16 = vaddq_s16(sum_s16, diff_lo_s16);
      sum_s16 = vaddq_s16(sum_s16, diff_hi_s16);

      sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_low_s16(diff_lo_s16),
                             vget_low_s16(diff_lo_s16));
      sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_high_s16(diff_lo_s16),
                             vget_high_s16(diff_lo_s16));

      sse_hi_s32 = vmlal_s16(sse_hi_s32, vget_low_s16(diff_hi_s16),
                             vget_low_s16(diff_hi_s16));
      sse_hi_s32 = vmlal_s16(sse_hi_s32, vget_high_s16(diff_hi_s16),
                             vget_high_s16(diff_hi_s16));
    }
    a += a_stride;
    b += b_stride;
  }

  *sum = horizontal_add_s16x8(sum_s16);
  *sse = (unsigned int)horizontal_add_s32x4(vaddq_s32(sse_lo_s32, sse_hi_s32));
}

// Process a block of width 8 two rows at a time.
static void variance_neon_w8x2(const uint8_t *a, int a_stride, const uint8_t *b,
                               int b_stride, int h, uint32_t *sse, int *sum) {
  int i = 0;
  int16x8_t sum_s16 = vdupq_n_s16(0);
  int32x4_t sse_lo_s32 = vdupq_n_s32(0);
  int32x4_t sse_hi_s32 = vdupq_n_s32(0);

  // Each column has it's own accumulator entry in sum_s16.
  assert(h <= 128);

  do {
    const uint8x8_t a_0_u8 = vld1_u8(a);
    const uint8x8_t a_1_u8 = vld1_u8(a + a_stride);
    const uint8x8_t b_0_u8 = vld1_u8(b);
    const uint8x8_t b_1_u8 = vld1_u8(b + b_stride);
    const uint16x8_t diff_0_u16 = vsubl_u8(a_0_u8, b_0_u8);
    const uint16x8_t diff_1_u16 = vsubl_u8(a_1_u8, b_1_u8);
    const int16x8_t diff_0_s16 = vreinterpretq_s16_u16(diff_0_u16);
    const int16x8_t diff_1_s16 = vreinterpretq_s16_u16(diff_1_u16);
    sum_s16 = vaddq_s16(sum_s16, diff_0_s16);
    sum_s16 = vaddq_s16(sum_s16, diff_1_s16);
    sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_low_s16(diff_0_s16),
                           vget_low_s16(diff_0_s16));
    sse_lo_s32 = vmlal_s16(sse_lo_s32, vget_low_s16(diff_1_s16),
{'fraction_non_alphanumeric': 0.09133450321718058, 'fraction_numerical': 0.10294977855770034, 'mean_word_length': 3.1426098996192455, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4199299', 'n_tokens_mistral': 6043, 'n_tokens_neox': 5465, 'n_words': 1027}