<commit_before>/*
 *  Copyright 2011 The LibYuv Project Authors. All rights reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS. All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "libyuv/compare.h"

#include <float.h>
#include <math.h>
#ifdef _OPENMP
#include <omp.h>
#endif

#include "libyuv/basic_types.h"
#include "libyuv/compare_row.h"
#include "libyuv/cpu_id.h"
#include "libyuv/row.h"
#include "libyuv/video_common.h"

#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif

// hash seed of 5381 recommended.
LIBYUV_API
uint32 HashDjb2(const uint8* src, uint64 count, uint32 seed) {
  const int kBlockSize = 1 << 15;  // 32768;
  int remainder;
  uint32 (*HashDjb2_SSE)(const uint8* src, int count, uint32 seed) = HashDjb2_C;
#if defined(HAS_HASHDJB2_SSE41)
  if (TestCpuFlag(kCpuHasSSE41)) {
    HashDjb2_SSE = HashDjb2_SSE41;
  }
#endif
#if defined(HAS_HASHDJB2_AVX2)
  if (TestCpuFlag(kCpuHasAVX2)) {
    HashDjb2_SSE = HashDjb2_AVX2;
  }
#endif

  while (count >= (uint64)(kBlockSize)) {
    seed = HashDjb2_SSE(src, kBlockSize, seed);
    src += kBlockSize;
    count -= kBlockSize;
  }
  remainder = (int)count & ~15;
  if (remainder) {
    seed = HashDjb2_SSE(src, remainder, seed);
    src += remainder;
    count -= remainder;
  }
  remainder = (int)count & 15;
  if (remainder) {
    seed = HashDjb2_C(src, remainder, seed);
  }
  return seed;
}

static uint32 ARGBDetectRow_C(const uint8* argb, int width) {
  int x;
  for (x = 0; x < width - 1; x += 2) {
    if (argb[0] != 255) {  // First byte is not Alpha of 255, so not ARGB.
      return FOURCC_BGRA;
    }
    if (argb[3] != 255) {  // 4th byte is not Alpha of 255, so not BGRA.
      return FOURCC_ARGB;
    }
    if (argb[4] != 255) {  // Second pixel first byte is not Alpha of 255.
      return FOURCC_BGRA;
    }
    if (argb[7] != 255) {  // Second pixel 4th byte is not Alpha of 255.
      return FOURCC_ARGB;
    }
    argb += 8;
  }
  if (width & 1) {
    if (argb[0] != 255) {  // First byte is not Alpha of 255, so not ARGB.
      return FOURCC_BGRA;
    }
    if (argb[3] != 255) {  // 4th byte is not Alpha of 255, so not BGRA.
      return FOURCC_ARGB;
    }
  }
  return 0;
}

// Scan an opaque argb image and return fourcc based on alpha offset.
// Returns FOURCC_ARGB, FOURCC_BGRA, or 0 if unknown.
LIBYUV_API
uint32 ARGBDetect(const uint8* argb, int stride_argb, int width, int height) {
  uint32 fourcc = 0;
  int h;

  // Coalesce rows.
  if (stride_argb == width * 4) {
    width *= height;
    height = 1;
    stride_argb = 0;
  }
  for (h = 0; h < height && fourcc == 0; ++h) {
    fourcc = ARGBDetectRow_C(argb, width);
    argb += stride_argb;
  }
  return fourcc;
}

LIBYUV_API
uint64 ComputeHammingDistance(const uint8* src_a,
                              const uint8* src_b,
                              int count) {
  const int kBlockSize = 65536;
  const int kSimdSize = 64;
  // SIMD for multiple of 64, and C for remainder
  int remainder = count & (kBlockSize - 1) & ~(kSimdSize - 1);
  uint64 diff = 0;
  int i;
  uint32 (*HammingDistance)(const uint8* src_a, const uint8* src_b, int count) =
      HammingDistance_C;
#if defined(HAS_HAMMINGDISTANCE_NEON)
  if (TestCpuFlag(kCpuHasNEON)) {
    HammingDistance = HammingDistance_NEON;
  }
#endif
#if defined(HAS_HAMMINGDISTANCE_X86)
  if (TestCpuFlag(kCpuHasX86)) {
    HammingDistance = HammingDistance_X86;
  }
#endif
#if defined(HAS_HAMMINGDISTANCE_SSSE3)
  if (TestCpuFlag(kCpuHasSSSE3)) {<commit_msg>Reduce HammingDistance block size to 32k to avoid overflow<commit_after>/*
 *  Copyright 2011 The LibYuv Project Authors. All rights reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS. All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "libyuv/compare.h"

#include <float.h>
#include <math.h>
#ifdef _OPENMP
#include <omp.h>
#endif

#include "libyuv/basic_types.h"
#include "libyuv/compare_row.h"
#include "libyuv/cpu_id.h"
#include "libyuv/row.h"
#include "libyuv/video_common.h"

#ifdef __cplusplus
namespace libyuv {
extern "C" {
#endif

// hash seed of 5381 recommended.
LIBYUV_API
uint32 HashDjb2(const uint8* src, uint64 count, uint32 seed) {
  const int kBlockSize = 1 << 15;  // 32768;
  int remainder;
  uint32 (*HashDjb2_SSE)(const uint8* src, int count, uint32 seed) = HashDjb2_C;
#if defined(HAS_HASHDJB2_SSE41)
  if (TestCpuFlag(kCpuHasSSE41)) {
    HashDjb2_SSE = HashDjb2_SSE41;
  }
#endif
#if defined(HAS_HASHDJB2_AVX2)
  if (TestCpuFlag(kCpuHasAVX2)) {
    HashDjb2_SSE = HashDjb2_AVX2;
  }
#endif

  while (count >= (uint64)(kBlockSize)) {
    seed = HashDjb2_SSE(src, kBlockSize, seed);
    src += kBlockSize;
    count -= kBlockSize;
  }
  remainder = (int)count & ~15;
  if (remainder) {
    seed = HashDjb2_SSE(src, remainder, seed);
    src += remainder;
    count -= remainder;
  }
  remainder = (int)count & 15;
  if (remainder) {
    seed = HashDjb2_C(src, remainder, seed);
  }
  return seed;
}

static uint32 ARGBDetectRow_C(const uint8* argb, int width) {
  int x;
  for (x = 0; x < width - 1; x += 2) {
    if (argb[0] != 255) {  // First byte is not Alpha of 255, so not ARGB.
      return FOURCC_BGRA;
    }
    if (argb[3] != 255) {  // 4th byte is not Alpha of 255, so not BGRA.
      return FOURCC_ARGB;
    }
    if (argb[4] != 255) {  // Second pixel first byte is not Alpha of 255.
      return FOURCC_BGRA;
    }
    if (argb[7] != 255) {  // Second pixel 4th byte is not Alpha of 255.
      return FOURCC_ARGB;
    }
    argb += 8;
  }
  if (width & 1) {
    if (argb[0] != 255) {  // First byte is not Alpha of 255, so not ARGB.
      return FOURCC_BGRA;
    }
    if (argb[3] != 255) {  // 4th byte is not Alpha of 255, so not BGRA.
      return FOURCC_ARGB;
    }
  }
  return 0;
}

// Scan an opaque argb image and return fourcc based on alpha offset.
// Returns FOURCC_ARGB, FOURCC_BGRA, or 0 if unknown.
LIBYUV_API
uint32 ARGBDetect(const uint8* argb, int stride_argb, int width, int height) {
  uint32 fourcc = 0;
  int h;

  // Coalesce rows.
  if (stride_argb == width * 4) {
    width *= height;
    height = 1;
    stride_argb = 0;
  }
  for (h = 0; h < height && fourcc == 0; ++h) {
    fourcc = ARGBDetectRow_C(argb, width);
    argb += stride_argb;
  }
  return fourcc;
}

// NEON version accumulates in 16 bit shorts which overflow at 65536 bytes.
// So actual maximum is 1 less loop, which is 64436 - 32 bytes.

LIBYUV_API
uint64 ComputeHammingDistance(const uint8* src_a,
                              const uint8* src_b,
                              int count) {
  const int kBlockSize = 1 << 15;  // 32768;
  const int kSimdSize = 64;
  // SIMD for multiple of 64, and C for remainder
  int remainder = count & (kBlockSize - 1) & ~(kSimdSize - 1);
  uint64 diff = 0;
  int i;
  uint32 (*HammingDistance)(const uint8* src_a, const uint8* src_b, int count) =
      HammingDistance_C;
#if defined(HAS_HAMMINGDISTANCE_NEON)
  if (TestCpuFlag(kCpuHasNEON)) {
    HammingDistance = HammingDistance_NEON;
  }
#endif
#if defined(HAS_HAMMINGDISTANCE_X86)
  if (TestCpuFlag(kCpuHasX86)) {
    HammingDistance = HammingDistance_X86;
  }
#endif
#if defined(HAS_HAMMINGDISTANCE_SSSE3)
  if (TestCpuFlag(kCpuHasSSSE3)) {
{'fraction_non_alphanumeric': 0.11032587357675697, 'fraction_numerical': 0.04214108101033896, 'mean_word_length': 3.0157645822385706, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3662942', 'n_tokens_mistral': 3174, 'n_tokens_neox': 2886, 'n_words': 866}