<commit_before>
#include "chrono_parallel/math/real3.h"
#include "chrono_parallel/math/sse.h"

#define set_m128r(lo, hi) _mm256_insertf128_ps(_mm256_castps128_ps256(lo), (hi), 1)

template <int i0, int i1, int i2, int i3>
static inline __m128i constant4i() {
    static const union {
        int i[4];
        __m128i xmm;
    } u = {{i0, i1, i2, i3}};
    return u.xmm;
}

// permute vector Vec2d
template <int i0, int i1>
static inline __m128d permute2d(__m128d const& a) {
    // is shuffling needed
    const bool do_shuffle = (i0 > 0) || (i1 != 1 && i1 >= 0);
    // is zeroing needed
    const bool do_zero = ((i0 | i1) < 0 && (i0 | i1) & 0x80);

    if (do_zero && !do_shuffle) {  // zeroing, not shuffling
        if ((i0 & i1) < 0)
            return _mm_setzero_pd();  // zero everything
        // zero some elements
        __m128i mask1 = constant4i<-int(i0 >= 0), -int(i0 >= 0), -int(i1 >= 0), -int(i1 >= 0)>();
        return _mm_and_pd(a, _mm_castsi128_pd(mask1));  // zero with AND mask
    } else if (do_shuffle && !do_zero) {                // shuffling, not zeroing
        return _mm_shuffle_pd(a, a, (i0 & 1) | (i1 & 1) << 1);
    } else if (do_shuffle && do_zero) {  // shuffling and zeroing
        // both shuffle and zero
        if (i0 < 0 && i1 >= 0) {  // zero low half, shuffle high half
            return _mm_shuffle_pd(_mm_setzero_pd(), a, (i1 & 1) << 1);
        }
        if (i0 >= 0 && i1 < 0) {  // shuffle low half, zero high half
            return _mm_shuffle_pd(a, _mm_setzero_pd(), i0 & 1);
        }
    }
    return a;  // trivial case: do nothing
}

// blend vectors Vec2d
template <int i0, int i1>
static inline __m128d blend2d(__m128d const& a, __m128d const& b) {
    // Combine all the indexes into a single bitfield, with 8 bits for each
    const int m1 = (i0 & 3) | (i1 & 3) << 8;

    // Mask to zero out negative indexes
    const int m2 = (i0 < 0 ? 0 : 0xFF) | (i1 < 0 ? 0 : 0xFF) << 8;

    if ((m1 & 0x0202 & m2) == 0) {
        // no elements from b, only elements from a and possibly zero
        return permute2d<i0, i1>(a);
    }
    if (((m1 ^ 0x0202) & 0x0202 & m2) == 0) {
        // no elements from a, only elements from b and possibly zero
        return permute2d<i0 & ~2, i1 & ~2>(b);
    }
    // selecting from both a and b without zeroing
    if ((i0 & 2) == 0) {  // first element from a, second element from b
        return _mm_shuffle_pd(a, b, (i0 & 1) | (i1 & 1) << 1);
    } else {  // first element from b, second element from a
        return _mm_shuffle_pd(b, a, (i0 & 1) | (i1 & 1) << 1);
    }
}

template <int i0, int i1, int i2, int i3, int i4, int i5, int i6, int i7>
static inline __m256 constant8f() {
    static const union {
        int i[8];
        __m256 ymm;
    } u = {{i0, i1, i2, i3, i4, i5, i6, i7}};
    return u.ymm;
}

template <int i0, int i1, int i2, int i3>
static inline __m256 permute4d(__m256 const& a) {
    const int ior = i0 | i1 | i2 | i3;  // OR indexes

    // is zeroing needed
    const bool do_zero = ior < 0 && (ior & 0x80);  // at least one index is negative, and not -0x100

    // is shuffling needed
    const bool do_shuffle = (i0 > 0) || (i1 != 1 && i1 >= 0) || (i2 != 2 && i2 >= 0) || (i3 != 3 && i3 >= 0);

    if (!do_shuffle) {  // no shuffling needed
        if (do_zero) {  // zeroing
            if ((i0 & i1 & i2 & i3) < 0) {
                return _mm256_setzero_pd();  // zero everything
            }
            // zero some elements
            __m256d const mask =
                _mm256_castps_pd(constant8f<-int(i0 >= 0), -int(i0 >= 0), -int(i1 >= 0), -int(i1 >= 0), -int(i2 >= 0),
                                            -int(i2 >= 0), -int(i3 >= 0), -int(i3 >= 0)>());
            return _mm256_and_pd(a, mask);  // zero with AND mask
        } else {
            return a;  // do nothing
        }
    }

    // Needed contents of low/high part of each source register in VSHUFPD
    // 0: a.low, 1: a.high, 3: zero
    const int s1 = (i0 < 0 ? 3 : (i0 & 2) >> 1) | (i2 < 0 ? 0x30 : (i2 & 2) << 3);
    const int s2 = (i1 < 0 ? 3 : (i1 & 2) >> 1) | (i3 < 0 ? 0x30 : (i3 & 2) << 3);
    // permute mask
    const int sm = (i0 < 0 ? 0 : (i0 & 1)) | (i1 < 0 ? 1 : (i1 & 1)) << 1 | (i2 < 0 ? 0 : (i2 & 1)) << 2 |
                   (i3 < 0 ? 1 : (i3 & 1)) << 3;

    if (s1 == 0x01 || s1 == 0x11 || s2 == 0x01 || s2 == 0x11) {
        // too expensive to use 256 bit permute, split into two 128 bit permutes
        __m128d alo = _mm256_castps256_ps128(a);
        __m128d ahi = _mm256_extractf128_ps(a, 1);
        __m128d rlo = blend2d<i0, i1>(alo, ahi);
        __m128d rhi = blend2d<i2, i3>(alo, ahi);
        return _mm256_castps_pd(set_m128r(_mm_castpd_ps(rlo), _mm_castpd_ps(rhi)));
    }

    // make operands for VSHUFPD
    __m256d r1, r2;

    switch (s1) {
        case 0x00:  // LL
            r1 = _mm256_insertf128_pd(a, _mm256_castpd256_pd128(a), 1);
            break;
        case 0x03:  // LZ
            r1 = _mm256_insertf128_pd(do_zero ? _mm256_setzero_pd() : __m256d(a), _mm256_castpd256_pd128(a), 1);
            break;
        case 0x10:  // LH
            r1 = a;
            break;
        case 0x13:  // ZH
            r1 = do_zero ? _mm256_and_pd(a, _mm256_castps_pd(constant8f<0, 0, 0, 0, -1, -1, -1, -1>())) : __m256d(a);
            break;
        case 0x30:  // LZ
            if (do_zero) {
                __m128d t = _mm256_castpd256_pd128(a);
                t = _mm_and_pd(t, t);
                r1 = _mm256_castpd128_pd256(t);
            } else
                r1 = a;
            break;
        case 0x31:  // HZ
            r1 = _mm256_castpd128_pd256(_mm256_extractf128_pd(a, 1));
            break;
        case 0x33:  // ZZ
            r1 = do_zero ? _mm256_setzero_pd() : __m256d(a);
            break;
    }

    if (s2 == s1) {
        if (sm == 0x0A)
            return r1;
        r2 = r1;
    } else {
        switch (s2) {
            case 0x00:  // LL
                r2 = _mm256_insertf128_pd(a, _mm256_castpd256_pd128(a), 1);
                break;
            case 0x03:  // ZL
                r2 = _mm256_insertf128_pd(do_zero ? _mm256_setzero_pd() : __m256d(a), _mm256_castpd256_pd128(a), 1);
                break;
            case 0x10:  // LH
                r2 = a;
                break;
            case 0x13:  // ZH
                r2 =
                    do_zero ? _mm256_and_pd(a, _mm256_castps_pd(constant8f<0, 0, 0, 0, -1, -1, -1, -1>())) : __m256d(a);
                break;
            case 0x30:  // LZ
                if (do_zero) {
                    __m128d t = _mm256_castpd256_pd128(a);
                    t = _mm_and_pd(t, t);
                    r2 = _mm256_castpd128_pd256(t);
                } else
                    r2 = a;
                break;
            case 0x31:  // HZ
                r2 = _mm256_castpd128_pd256(_mm256_extractf128_pd(a, 1));
                break;
            case 0x33:  // ZZ
                r2 = do_zero ? _mm256_setzero_pd() : __m256d(a);
                break;
        }
    }
    return _mm256_shuffle_pd(r1, r2, sm);
}

namespace chrono {
<commit_msg>put code for real3 in ifdef that requires avx<commit_after>
#include "chrono_parallel/math/real3.h"
#include "chrono_parallel/math/sse.h"
#if defined(CHRONO_PARALLEL_USE_SIMD) && defined(CHRONO_PARALLEL_HAS_AVX) && defined(CHRONO_PARALLEL_USE_DOUBLE)
#define set_m128r(lo, hi) _mm256_insertf128_ps(_mm256_castps128_ps256(lo), (hi), 1)

template <int i0, int i1, int i2, int i3>
static inline __m128i constant4i() {
    static const union {
        int i[4];
        __m128i xmm;
    } u = {{i0, i1, i2, i3}};
    return u.xmm;
}

// permute vector Vec2d
template <int i0, int i1>
static inline __m128d permute2d(__m128d const& a) {
    // is shuffling needed
    const bool do_shuffle = (i0 > 0) || (i1 != 1 && i1 >= 0);
    // is zeroing needed
    const bool do_zero = ((i0 | i1) < 0 && (i0 | i1) & 0x80);

    if (do_zero && !do_shuffle) {  // zeroing, not shuffling
        if ((i0 & i1) < 0)
            return _mm_setzero_pd();  // zero everything
        // zero some elements
        __m128i mask1 = constant4i<-int(i0 >= 0), -int(i0 >= 0), -int(i1 >= 0), -int(i1 >= 0)>();
        return _mm_and_pd(a, _mm_castsi128_pd(mask1));  // zero with AND mask
    } else if (do_shuffle && !do_zero) {                // shuffling, not zeroing
        return _mm_shuffle_pd(a, a, (i0 & 1) | (i1 & 1) << 1);
    } else if (do_shuffle && do_zero) {  // shuffling and zeroing
        // both shuffle and zero
        if (i0 < 0 && i1 >= 0) {  // zero low half, shuffle high half
            return _mm_shuffle_pd(_mm_setzero_pd(), a, (i1 & 1) << 1);
        }
        if (i0 >= 0 && i1 < 0) {  // shuffle low half, zero high half
            return _mm_shuffle_pd(a, _mm_setzero_pd(), i0 & 1);
        }
    }
    return a;  // trivial case: do nothing
}

// blend vectors Vec2d
template <int i0, int i1>
static inline __m128d blend2d(__m128d const& a, __m128d const& b) {
    // Combine all the indexes into a single bitfield, with 8 bits for each
    const int m1 = (i0 & 3) | (i1 & 3) << 8;

    // Mask to zero out negative indexes
    const int m2 = (i0 < 0 ? 0 : 0xFF) | (i1 < 0 ? 0 : 0xFF) << 8;

    if ((m1 & 0x0202 & m2) == 0) {
        // no elements from b, only elements from a and possibly zero
        return permute2d<i0, i1>(a);
    }
    if (((m1 ^ 0x0202) & 0x0202 & m2) == 0) {
        // no elements from a, only elements from b and possibly zero
        return permute2d<i0 & ~2, i1 & ~2>(b);
    }
    // selecting from both a and b without zeroing
    if ((i0 & 2) == 0) {  // first element from a, second element from b
        return _mm_shuffle_pd(a, b, (i0 & 1) | (i1 & 1) << 1);
    } else {  // first element from b, second element from a
        return _mm_shuffle_pd(b, a, (i0 & 1) | (i1 & 1) << 1);
    }
}

template <int i0, int i1, int i2, int i3, int i4, int i5, int i6, int i7>
static inline __m256 constant8f() {
    static const union {
        int i[8];
        __m256 ymm;
    } u = {{i0, i1, i2, i3, i4, i5, i6, i7}};
    return u.ymm;
}

template <int i0, int i1, int i2, int i3>
static inline __m256 permute4d(__m256 const& a) {
    const int ior = i0 | i1 | i2 | i3;  // OR indexes

    // is zeroing needed
    const bool do_zero = ior < 0 && (ior & 0x80);  // at least one index is negative, and not -0x100

    // is shuffling needed
    const bool do_shuffle = (i0 > 0) || (i1 != 1 && i1 >= 0) || (i2 != 2 && i2 >= 0) || (i3 != 3 && i3 >= 0);

    if (!do_shuffle) {  // no shuffling needed
        if (do_zero) {  // zeroing
            if ((i0 & i1 & i2 & i3) < 0) {
                return _mm256_setzero_pd();  // zero everything
            }
            // zero some elements
            __m256d const mask =
                _mm256_castps_pd(constant8f<-int(i0 >= 0), -int(i0 >= 0), -int(i1 >= 0), -int(i1 >= 0), -int(i2 >= 0),
                                            -int(i2 >= 0), -int(i3 >= 0), -int(i3 >= 0)>());
            return _mm256_and_pd(a, mask);  // zero with AND mask
        } else {
            return a;  // do nothing
        }
    }

    // Needed contents of low/high part of each source register in VSHUFPD
    // 0: a.low, 1: a.high, 3: zero
    const int s1 = (i0 < 0 ? 3 : (i0 & 2) >> 1) | (i2 < 0 ? 0x30 : (i2 & 2) << 3);
    const int s2 = (i1 < 0 ? 3 : (i1 & 2) >> 1) | (i3 < 0 ? 0x30 : (i3 & 2) << 3);
    // permute mask
    const int sm = (i0 < 0 ? 0 : (i0 & 1)) | (i1 < 0 ? 1 : (i1 & 1)) << 1 | (i2 < 0 ? 0 : (i2 & 1)) << 2 |
                   (i3 < 0 ? 1 : (i3 & 1)) << 3;

    if (s1 == 0x01 || s1 == 0x11 || s2 == 0x01 || s2 == 0x11) {
        // too expensive to use 256 bit permute, split into two 128 bit permutes
        __m128d alo = _mm256_castps256_ps128(a);
        __m128d ahi = _mm256_extractf128_ps(a, 1);
        __m128d rlo = blend2d<i0, i1>(alo, ahi);
        __m128d rhi = blend2d<i2, i3>(alo, ahi);
        return _mm256_castps_pd(set_m128r(_mm_castpd_ps(rlo), _mm_castpd_ps(rhi)));
    }

    // make operands for VSHUFPD
    __m256d r1, r2;

    switch (s1) {
        case 0x00:  // LL
            r1 = _mm256_insertf128_pd(a, _mm256_castpd256_pd128(a), 1);
            break;
        case 0x03:  // LZ
            r1 = _mm256_insertf128_pd(do_zero ? _mm256_setzero_pd() : __m256d(a), _mm256_castpd256_pd128(a), 1);
            break;
        case 0x10:  // LH
            r1 = a;
            break;
        case 0x13:  // ZH
            r1 = do_zero ? _mm256_and_pd(a, _mm256_castps_pd(constant8f<0, 0, 0, 0, -1, -1, -1, -1>())) : __m256d(a);
            break;
        case 0x30:  // LZ
            if (do_zero) {
                __m128d t = _mm256_castpd256_pd128(a);
                t = _mm_and_pd(t, t);
                r1 = _mm256_castpd128_pd256(t);
            } else
                r1 = a;
            break;
        case 0x31:  // HZ
            r1 = _mm256_castpd128_pd256(_mm256_extractf128_pd(a, 1));
            break;
        case 0x33:  // ZZ
            r1 = do_zero ? _mm256_setzero_pd() : __m256d(a);
            break;
    }

    if (s2 == s1) {
        if (sm == 0x0A)
            return r1;
        r2 = r1;
    } else {
        switch (s2) {
            case 0x00:  // LL
                r2 = _mm256_insertf128_pd(a, _mm256_castpd256_pd128(a), 1);
                break;
            case 0x03:  // ZL
                r2 = _mm256_insertf128_pd(do_zero ? _mm256_setzero_pd() : __m256d(a), _mm256_castpd256_pd128(a), 1);
                break;
            case 0x10:  // LH
                r2 = a;
                break;
            case 0x13:  // ZH
                r2 =
                    do_zero ? _mm256_and_pd(a, _mm256_castps_pd(constant8f<0, 0, 0, 0, -1, -1, -1, -1>())) : __m256d(a);
                break;
            case 0x30:  // LZ
                if (do_zero) {
                    __m128d t = _mm256_castpd256_pd128(a);
                    t = _mm_and_pd(t, t);
                    r2 = _mm256_castpd128_pd256(t);
                } else
                    r2 = a;
                break;
            case 0x31:  // HZ
                r2 = _mm256_castpd128_pd256(_mm256_extractf128_pd(a, 1));
                break;
            case 0x33:  // ZZ
                r2 = do_zero ? _mm256_setzero_pd() : __m256d(a);
                break;
        }
    }
    return _mm256_shuffle_pd(r1, r2, sm);
}
#endif

namespace chrono {

{'fraction_non_alphanumeric': 0.13813730263615498, 'fraction_numerical': 0.0916046463100786, 'mean_word_length': 1.6940228592842421, 'pattern_counts': {'":': 0, '<': 105, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5430930', 'n_tokens_mistral': 6367, 'n_tokens_neox': 5544, 'n_words': 1533}