<commit_before>// libpopcnt.h - C/C++ library for counting the number of 1 bits (bit
// population count) in an array as quickly as possible using
// specialized CPU instructions e.g. POPCNT, AVX2.
//
// Copyright (c) 2016, <NAME>
// Copyright (c) 2016, <NAME>
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
//    list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
//    this list of conditions and the following disclaimer in the documentation
//    and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#ifndef LIBPOPCNT_H
#define LIBPOPCNT_H

#include <stdint.h>

#if defined(HAVE_POPCNT) || \
    defined(HAVE_AVX2)

#if defined(_MSC_VER) && \
   (defined(_WIN32) || defined(_WIN64))
  // __cpuidex()
  #include <intrin.h>
#endif

// %ebx bit flags
#define bit_AVX2 (1 << 5)

// %ecx bit flags
#define bit_POPCNT (1 << 23)

/// Portable cpuid implementation for x86 and x86-64 CPUs
/// (supports PIC and non-PIC code).
/// Returns 1 if the CPU supports cpuid else 0.
///
static int cpuid(unsigned int *eax,
                 unsigned int *ebx,
                 unsigned int *ecx,
                 unsigned int *edx)
{
#if defined(_MSC_VER) && \
   (defined(_WIN32) || defined(_WIN64))
  int regs[4];
  __cpuidex(regs, *eax, *ecx);
  *eax = regs[0];
  *ebx = regs[1];
  *ecx = regs[2];
  *edx = regs[3];
  return 1;
#elif defined(__i386__) || defined(__i386)
  #if defined(__PIC__)
    __asm__ __volatile__ (
     "mov %%ebx, %%esi;" // save %ebx PIC register
     "cpuid;"
     "xchg %%ebx, %%esi;"
     : "+a" (*eax), 
       "+S" (*ebx),
       "+c" (*ecx),
       "+d" (*edx));
  #else
    __asm__ __volatile__ (
     "cpuid;"
     : "+a" (*eax), 
       "+b" (*ebx),
       "+c" (*ecx),
       "+d" (*edx));
  #endif
  return 1;
#elif defined(__x86_64__)
  __asm__ __volatile__ (
   "cpuid;"
   : "+a" (*eax), 
     "+b" (*ebx),
     "+c" (*ecx),
     "+d" (*edx));
  return 1;
#else
  (void) eax;
  (void) ebx;
  (void) ecx;
  (void) edx;
  return 0;
#endif
}

#endif

#if defined(HAVE_POPCNT)

static int init_has_popcnt()
{
  unsigned int eax = 1;
  unsigned int ebx;
  unsigned int ecx = 0;
  unsigned int edx;

  if (cpuid(&eax, &ebx, &ecx, &edx) != -1)
  {
    return (ecx & bit_POPCNT) != 0;
  }

  return 0;
}

static int has_popcnt()
{
  static int popcnt = init_has_popcnt();
  return popcnt;
}

#endif /* HAVE_POPCNT */

#if defined(HAVE_AVX2)

static int init_has_avx2()
{
  unsigned int eax = 7;
  unsigned int ebx;
  unsigned int ecx = 0;
  unsigned int edx;

  if (cpuid(&eax, &ebx, &ecx, &edx) != -1)<commit_msg>Support non x86 CPUs<commit_after>// libpopcnt.h - C/C++ library for counting the number of 1 bits (bit
// population count) in an array as quickly as possible using
// specialized CPU instructions e.g. POPCNT, AVX2.
//
// Copyright (c) 2016, <NAME>
// Copyright (c) 2016, <NAME>
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
//    list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
//    this list of conditions and the following disclaimer in the documentation
//    and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#ifndef LIBPOPCNT_H
#define LIBPOPCNT_H

#include <stdint.h>

#if (defined(__i386__) || \
     defined(__x86_64__) || \
     defined(_WIN32) || \
     defined(_WIN64)) && \
    (defined(HAVE_POPCNT) || defined(HAVE_AVX2))

#if defined(_MSC_VER) && \
   (defined(_WIN32) || defined(_WIN64))
  // __cpuidex()
  #include <intrin.h>
#endif

// %ebx bit flags
#define bit_AVX2 (1 << 5)

// %ecx bit flags
#define bit_POPCNT (1 << 23)

/// Portable cpuid implementation for x86 and x86-64 CPUs
/// (supports PIC and non-PIC code).
/// Returns 1 if the CPU supports cpuid else 0.
///
static int cpuid(unsigned int *eax,
                 unsigned int *ebx,
                 unsigned int *ecx,
                 unsigned int *edx)
{
#if defined(_MSC_VER) && \
   (defined(_WIN32) || defined(_WIN64))
  int regs[4];
  __cpuidex(regs, *eax, *ecx);
  *eax = regs[0];
  *ebx = regs[1];
  *ecx = regs[2];
  *edx = regs[3];
  return 1;
#elif defined(__i386__)
  #if defined(__PIC__)
    __asm__ __volatile__ (
     "mov %%ebx, %%esi;" // save %ebx PIC register
     "cpuid;"
     "xchg %%ebx, %%esi;"
     : "+a" (*eax), 
       "+S" (*ebx),
       "+c" (*ecx),
       "+d" (*edx));
  #else
    __asm__ __volatile__ (
     "cpuid;"
     : "+a" (*eax), 
       "+b" (*ebx),
       "+c" (*ecx),
       "+d" (*edx));
  #endif
  return 1;
#elif defined(__x86_64__)
  __asm__ __volatile__ (
   "cpuid;"
   : "+a" (*eax), 
     "+b" (*ebx),
     "+c" (*ecx),
     "+d" (*edx));
  return 1;
#else
  (void) eax;
  (void) ebx;
  (void) ecx;
  (void) edx;
  return 0;
#endif
}

#endif

#if defined(HAVE_POPCNT)

// x86 & x86_64 CPUs
#if (defined(__i386__) || \
     defined(__x86_64__) || \
     defined(_WIN32) || \
     defined(_WIN64))

static int init_has_popcnt()
{
  unsigned int eax = 1;
  unsigned int ebx;
  unsigned int ecx = 0;
  unsigned int edx;

  if (cpuid(&eax, &ebx, &ecx, &edx) != -1)
  {
    return (ecx & bit_POPCNT) != 0;
  }

  return 0;
}

static int has_popcnt()
{
  static int popcnt = init_has_popcnt();
  return popcnt;
}

#else

static int has_popcnt()
{
  // On non x86 CPUs use the POPCNT instruction
  // without runtime check
  return 1;
}

#endif

#endif /* HAVE_POPCNT */

#if defined(HAVE_AVX2)

static int init_has_avx2()
{
  unsigned int eax = 7;
  unsigned int ebx;
  unsigned int ecx = 0;
  unsigned int edx;

  if (cpuid(&eax, &ebx, &ecx, &edx) != -1)
{'fraction_non_alphanumeric': 0.13139167105956773, 'fraction_numerical': 0.020822351080653664, 'mean_word_length': 3.043153969099627, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1587264', 'n_tokens_mistral': 2827, 'n_tokens_neox': 2557, 'n_words': 923}