<commit_before>/**
 * This is code is released under the
 * Apache License Version 2.0 http://www.apache.org/licenses/.
 *
 * (c) <NAME>, http://lemire.me/en/
 */

#ifndef EWAHUTIL_H
#define EWAHUTIL_H

#include <string.h>
#include <stdlib.h>
#include <iso646.h> // mostly for Microsoft compilers



#if defined __UINT32_MAX__ or UINT32_MAX
#include <inttypes.h>
#else
typedef unsigned short uint16_t;
typedef unsigned long uint32_t;
typedef unsigned long long uint64_t;
#endif
#include <cassert>
#include <iostream>
#include <vector>
#include <string>
#include <stdexcept>
#include <cstddef>
#include <algorithm>
#include <sstream>

// taken from stackoverflow
#ifndef NDEBUG
#   define ASSERT(condition, message) \
    do { \
        if (! (condition)) { \
            std::cerr << "Assertion `" #condition "` failed in " << __FILE__ \
                      << " line " << __LINE__ << ": " << message << std::endl; \
            std::exit(EXIT_FAILURE); \
        } \
    } while (false)
#else
#   define ASSERT(condition, message) do { } while (false)
#endif


#ifdef _MSC_VER
#include <intrin.h>
/**
 * count the number of bits set to one (32 bit version)
 */
uint32_t countOnes(uint32_t x) {
    return __popcnt((x));
}
#else
/**
 * count the number of bits set to one (32 bit version)
 */
uint32_t countOnes(uint32_t x) {
    return static_cast<uint32_t>(__builtin_popcount(x));
}
#endif


/**
 * count the number of bits set to one (64 bit version)
 */
uint32_t countOnes(uint64_t v) {
    return countOnes(static_cast<uint32_t> (v)) + countOnes(
            static_cast<uint32_t> (v >> 32));
}

uint32_t countOnes(uint16_t v) {
    return countOnes(static_cast<uint32_t>(v));
}

#ifdef _MSC_VER
#include <intrin.h>

// contributed by <NAME>
int numberOfTrailingZeros(uint32_t x) {
    unsigned long r = 0;
    if(_BitScanReverse(&r, x))// specific to MS compilers
        return 32;
    return r;
}
#else
int numberOfTrailingZeros(uint32_t x) {
    if (x == 0) return 32;
    return __builtin_ctz(x);// specific to GCC
}
#endif

int numberOfTrailingZeros(uint64_t x) {
    if(static_cast<uint32_t> (x)!= 0) {
        return numberOfTrailingZeros(static_cast<uint32_t> (x));
    }
    else return 32+numberOfTrailingZeros(static_cast<uint32_t> (x >> 32));
}

#ifdef _MSC_VER
// contributed by <NAME>
int numberOfTrailingZeros(uint16_t x) {
  unsigned long r = 0;
  if(x == 0) return 16;
  _BitScanReverse(&r, static_cast<uint32_t>(x));// specific to MS compilers
  return r;
}
#else
int numberOfTrailingZeros(uint16_t x) {
    if (x == 0) return 16;
    return __builtin_ctz(x);// specific to GCC
}
#endif

/**
 * Returns the binary representation of a binary word.
 */
template<class uword>
std::string toBinaryString(const uword w) {
    std::ostringstream convert;
    for (uint32_t k = 0; k < sizeof(uword) * 8; ++k) {
        if (w & (static_cast<uword> (1) << k))
            convert << "1";
        else
            convert << "0";
    }
    return convert.str();
}

#endif<commit_msg>Trying to fix a bug with microsoft compilers, without access to a microsoft compiler.<commit_after>/**
 * This is code is released under the
 * Apache License Version 2.0 http://www.apache.org/licenses/.
 *
 * (c) <NAME>, http://lemire.me/en/
 */

#ifndef EWAHUTIL_H
#define EWAHUTIL_H

#include <string.h>
#include <stdlib.h>
#include <iso646.h> // mostly for Microsoft compilers



#if defined __UINT32_MAX__ or UINT32_MAX
#include <inttypes.h>
#else
typedef unsigned short uint16_t;
typedef unsigned long uint32_t;
typedef unsigned long long uint64_t;
#endif
#include <cassert>
#include <iostream>
#include <vector>
#include <string>
#include <stdexcept>
#include <cstddef>
#include <algorithm>
#include <sstream>

// taken from stackoverflow
#ifndef NDEBUG
#   define ASSERT(condition, message) \
    do { \
        if (! (condition)) { \
            std::cerr << "Assertion `" #condition "` failed in " << __FILE__ \
                      << " line " << __LINE__ << ": " << message << std::endl; \
            std::exit(EXIT_FAILURE); \
        } \
    } while (false)
#else
#   define ASSERT(condition, message) do { } while (false)
#endif


#ifdef _MSC_VER
#include <intrin.h>
/**
 * count the number of bits set to one (32 bit version)
 */
uint32_t countOnes(uint32_t x) {
    return __popcnt((x));
}
#else
/**
 * count the number of bits set to one (32 bit version)
 */
uint32_t countOnes(uint32_t x) {
    return static_cast<uint32_t>(__builtin_popcount(x));
}
#endif


/**
 * count the number of bits set to one (64 bit version)
 */
uint32_t countOnes(uint64_t v) {
    return countOnes(static_cast<uint32_t> (v)) + countOnes(
            static_cast<uint32_t> (v >> 32));
}

uint32_t countOnes(uint16_t v) {
    return countOnes(static_cast<uint32_t>(v));
}

#ifdef _MSC_VER
#include <intrin.h>

// contributed by <NAME>
int numberOfTrailingZeros(uint32_t x) {
    unsigned long r = 0;
    if(_BitScanReverse(&r, x))// specific to MS compilers
        return 32;
    return r - 1;
}
#else
int numberOfTrailingZeros(uint32_t x) {
    if (x == 0) return 32;
    return __builtin_ctz(x);// specific to GCC
}
#endif

int numberOfTrailingZeros(uint64_t x) {
    if(static_cast<uint32_t> (x)!= 0) {
        return numberOfTrailingZeros(static_cast<uint32_t> (x));
    }
    else return 32+numberOfTrailingZeros(static_cast<uint32_t> (x >> 32));
}

#ifdef _MSC_VER
// contributed by <NAME>
int numberOfTrailingZeros(uint16_t x) {
  unsigned long r = 0;
  if(x == 0) return 16;
  _BitScanReverse(&r, static_cast<uint32_t>(x));// specific to MS compilers
  return r - 1;
}
#else
int numberOfTrailingZeros(uint16_t x) {
    if (x == 0) return 16;
    return __builtin_ctz(x);// specific to GCC
}
#endif

/**
 * Returns the binary representation of a binary word.
 */
template<class uword>
std::string toBinaryString(const uword w) {
    std::ostringstream convert;
    for (uint32_t k = 0; k < sizeof(uword) * 8; ++k) {
        if (w & (static_cast<uword> (1) << k))
            convert << "1";
        else
            convert << "0";
    }
    return convert.str();
}

#endif
{'fraction_non_alphanumeric': 0.13491280026324448, 'fraction_numerical': 0.029944060546232314, 'mean_word_length': 3.3514674302075877, 'pattern_counts': {'":': 2, '<': 99, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2179357', 'n_tokens_mistral': 2317, 'n_tokens_neox': 2174, 'n_words': 600}