<commit_before>/* Copyright 2012 <NAME> <<EMAIL>> */
#define LFR_SSE2 1

#include "cpu.h"
#if defined(LFR_CPU_X86)
#include "resample.h"
#include <string.h>

void
lfr_resample_s16n2s16_sse2(
    lfr_fixed_t *pos, lfr_fixed_t inv_ratio, unsigned *dither,
    void *out, int outlen, const void *in, int inlen,
    const struct lfr_filter *filter)
{
    const __m128i *fd;
    int flen, log2nfilt;
    lfr_fixed_t x;

    __m128i acc, acc0, acc1, fir0, fir1, fir_interp, dat0, dat1;
    __m128i dsv, lcg_a, lcg_c;
    int fn, ff0, ff1, off, fidx0, fidx1;
    int accs0, accs1, i, j, f;
    unsigned ds;

    union {
        unsigned d[4];
        __m128i x;
    } u;

    /* fd: Pointer to beginning of filter coefficients, aligned.  */
    fd = (const __m128i *) filter->data;
    /* flen: Length of filter, measured in 128-bit words.  */
    flen = filter->nsamp >> 3;
    /* log2nfilt: Base 2 logarithm of the number of filters.  */
    log2nfilt = filter->log2nfilt;
    x = *pos;

    ds = *dither;
    for (i = 0; i < 4; ++i) {
        u.d[i] = ds;
        ds = ds * LCG_A + LCG_C;
    }
    dsv = u.x;
    lcg_a = _mm_set1_epi32(LCG_A4);
    lcg_c = _mm_set1_epi32(LCG_C4);

    acc0 = _mm_set1_epi32(0);
    acc1 = _mm_set1_epi32(0);
    for (i = 0; i < outlen; ++i) {
        /* fn: filter number
           ff0: filter factor for filter fn
           ff1: filter factor for filter fn+1 */
        fn = (((unsigned) x >> 1) >> (31 - log2nfilt)) &
            ((1u << log2nfilt) - 1);
        ff1 = ((unsigned) x >> (32 - log2nfilt - INTERP_BITS)) &
            ((1u << INTERP_BITS) - 1);
        ff0 = (1u << INTERP_BITS) - ff1;
        fir_interp = _mm_set1_epi32(ff0 | (ff1 << 16));

        /* acc: FIR accumulator, used for accumulating 32-bit values
           in the format L R L R.  This corresponds to one frame of
           output, so the pair of values for left and the pair for
           right have to be summed later.  */
        acc = _mm_set1_epi32(0);
        /* off: offset in input corresponding to first sample in
           filter */
        off = (int) (x >> 32);
        /* fixd0, fidx1: start, end indexes of 8-word (16-byte) chunks
           of whole FIR data we will use */
        fidx0 = (-off + 7) >> 3;
        fidx1 = (inlen - off) >> 3;
        if (fidx0 > 0) {
            if (fidx0 > flen)
                goto accumulate;
            accs0 = 0;
            accs1 = 0;
            for (j = -off; j < fidx0 * 8; ++j) {
                f = (((const short *) fd)[(fn+0) * (flen*8) + j] * ff0 +
                     ((const short *) fd)[(fn+1) * (flen*8) + j] * ff1)
                    >> INTERP_BITS;
                accs0 += ((const short *) in)[(j + off)*2 + 0] * f;
                accs1 += ((const short *) in)[(j + off)*2 + 1] * f;
            }
            acc = _mm_set_epi32(0, 0, accs1, accs0);
        } else {
            fidx0 = 0;
        }
        if (fidx1 < flen) {
            if (fidx1 < 0)
                goto accumulate;
            accs0 = 0;
            accs1 = 0;
            for (j = fidx1 * 8; j < inlen - off; ++j) {
                f = (((const short *) fd)[(fn+0) * (flen*8) + j] * ff0 +
                     ((const short *) fd)[(fn+1) * (flen*8) + j] * ff1)
                    >> INTERP_BITS;
                accs0 += ((const short *) in)[(j + off)*2 + 0] * f;
                accs1 += ((const short *) in)[(j + off)*2 + 1] * f;
            }
            acc = _mm_add_epi32(acc, _mm_set_epi32(0, 0, accs1, accs0));
        } else {
            fidx1 = flen;
        }

        for (j = fidx0; j < fidx1; ++j) {
            dat0 = _mm_loadu_si128(
                (const __m128i *)
                ((const short *) in + off*2 + j*16));
            dat1 = _mm_loadu_si128(
                (const __m128i *)
                ((const short *) in + off*2 + j*16+8));
            fir0 = fd[(fn+0)*flen + j];
            fir1 = fd[(fn+1)*flen + j];

            fir0 = _mm_packs_epi32(
                _mm_srai_epi32(
                    _mm_madd_epi16(
                        _mm_unpacklo_epi16(fir0, fir1),
                        fir_interp),
                    INTERP_BITS),
                _mm_srai_epi32(
                    _mm_madd_epi16(
                        _mm_unpackhi_epi16(fir0, fir1),
                        fir_interp),
                    INTERP_BITS));

            fir1 = _mm_unpackhi_epi16(fir0, fir0);
            fir0 = _mm_unpacklo_epi16(fir0, fir0);
            acc = _mm_add_epi32(
                acc,
                _mm_add_epi32(
                    _mm_madd_epi16(
                        _mm_unpacklo_epi16(dat0, dat1),
                        _mm_unpacklo_epi16(fir0, fir1)),
                    _mm_madd_epi16(
                        _mm_unpackhi_epi16(dat0, dat1),
                        _mm_unpackhi_epi16(fir0, fir1))));
        }

    accumulate:
        switch (i & 3) {
        case 0: case 2:
            acc0 = acc;
            break;

        case 1:
            acc1 = _mm_add_epi32(
                _mm_unpacklo_epi64(acc0, acc),
                _mm_unpackhi_epi64(acc0, acc));
            break;

        case 3:
            acc0 = _mm_add_epi32(
                _mm_unpacklo_epi64(acc0, acc),
                _mm_unpackhi_epi64(acc0, acc));

            /* Apply dither */
            acc1 = _mm_add_epi32(acc1, _mm_srli_epi32(dsv, 17));
            dsv = lfr_rand_epu32(dsv, lcg_a, lcg_c);
            acc0 = _mm_add_epi32(acc0, _mm_srli_epi32(dsv, 17));
            dsv = lfr_rand_epu32(dsv, lcg_a, lcg_c);

            acc = _mm_packs_epi32(
                _mm_srai_epi32(acc1, 15),
                _mm_srai_epi32(acc0, 15));
            _mm_storeu_si128((__m128i *) ((short *) out + (i - 3) * 2), acc);
            break;
        }

        x += inv_ratio;
    }

    ds = _mm_cvtsi128_si32(dsv);
    for (i = 0; i < (outlen & 3) * 2; ++i)
        ds = LCG_A * ds + LCG_C;
    *pos = x;
    *dither = ds;

    /* Store remaing bytes */
    acc = _mm_set1_epi32(0);
    if ((i & 3) == 0)
        return;
    for (i = outlen; ; ++i) {
        switch (i & 3) {
        case 0: case 2:
            acc0 = acc;
            break;

        case 1:
            acc1 = _mm_add_epi32(
                _mm_unpacklo_epi64(acc0, acc),
                _mm_unpackhi_epi64(acc0, acc));
            break;

        case 3:<commit_msg>Unroll loops in s16n2s16_sse2<commit_after>/* Copyright 2012 <NAME> <<EMAIL>> */
#define LFR_SSE2 1

#include "cpu.h"
#if defined(LFR_CPU_X86)
#include "resample.h"
#include <string.h>

void
lfr_resample_s16n2s16_sse2(
    lfr_fixed_t *pos, lfr_fixed_t inv_ratio, unsigned *dither,
    void *out, int outlen, const void *in, int inlen,
    const struct lfr_filter *filter)
{
    const __m128i *fd;
    int flen, log2nfilt;
    lfr_fixed_t x;

    __m128i acc, acc0, acc1, fir0, fir1, fir_interp, dat0, dat1;
    __m128i acc_a, acc_b;
    __m128i dsv, lcg_a, lcg_c;
    int fn, ff0, ff1, off, fidx0, fidx1;
    int accs0, accs1, i, j, f;
    unsigned ds;

    union {
        unsigned d[4];
        __m128i x;
    } u;

    /* fd: Pointer to beginning of filter coefficients, aligned.  */
    fd = (const __m128i *) filter->data;
    /* flen: Length of filter, measured in 128-bit words.  */
    flen = filter->nsamp >> 3;
    /* log2nfilt: Base 2 logarithm of the number of filters.  */
    log2nfilt = filter->log2nfilt;
    x = *pos;

    ds = *dither;
    for (i = 0; i < 4; ++i) {
        u.d[i] = ds;
        ds = ds * LCG_A + LCG_C;
    }
    dsv = u.x;
    lcg_a = _mm_set1_epi32(LCG_A4);
    lcg_c = _mm_set1_epi32(LCG_C4);

    acc0 = _mm_set1_epi32(0);
    acc1 = _mm_set1_epi32(0);
    switch (flen) {
    case 2: goto flen2;
    case 3: goto flen3;
    default: goto flenv;
    }

flenv:
    for (i = 0; i < outlen; ++i) {
        /* fn: filter number
           ff0: filter factor for filter fn
           ff1: filter factor for filter fn+1 */
        fn = (((unsigned) x >> 1) >> (31 - log2nfilt)) &
            ((1u << log2nfilt) - 1);
        ff1 = ((unsigned) x >> (32 - log2nfilt - INTERP_BITS)) &
            ((1u << INTERP_BITS) - 1);
        ff0 = (1u << INTERP_BITS) - ff1;
        fir_interp = _mm_set1_epi32(ff0 | (ff1 << 16));

        /* acc: FIR accumulator, used for accumulating 32-bit values
           in the format L R L R.  This corresponds to one frame of
           output, so the pair of values for left and the pair for
           right have to be summed later.  */
        acc = _mm_set1_epi32(0);
        /* off: offset in input corresponding to first sample in
           filter */
        off = (int) (x >> 32);
        /* fixd0, fidx1: start, end indexes of 8-word (16-byte) chunks
           of whole FIR data we will use */
        fidx0 = (-off + 7) >> 3;
        fidx1 = (inlen - off) >> 3;
        if (fidx0 > 0) {
            if (fidx0 > flen)
                goto flenv_accumulate;
            accs0 = 0;
            accs1 = 0;
            for (j = -off; j < fidx0 * 8; ++j) {
                f = (((const short *) fd)[(fn+0) * (flen*8) + j] * ff0 +
                     ((const short *) fd)[(fn+1) * (flen*8) + j] * ff1)
                    >> INTERP_BITS;
                accs0 += ((const short *) in)[(j + off)*2 + 0] * f;
                accs1 += ((const short *) in)[(j + off)*2 + 1] * f;
            }
            acc = _mm_set_epi32(0, 0, accs1, accs0);
        } else {
            fidx0 = 0;
        }
        if (fidx1 < flen) {
            if (fidx1 < 0)
                goto flenv_accumulate;
            accs0 = 0;
            accs1 = 0;
            for (j = fidx1 * 8; j < inlen - off; ++j) {
                f = (((const short *) fd)[(fn+0) * (flen*8) + j] * ff0 +
                     ((const short *) fd)[(fn+1) * (flen*8) + j] * ff1)
                    >> INTERP_BITS;
                accs0 += ((const short *) in)[(j + off)*2 + 0] * f;
                accs1 += ((const short *) in)[(j + off)*2 + 1] * f;
            }
            acc = _mm_add_epi32(acc, _mm_set_epi32(0, 0, accs1, accs0));
        } else {
            fidx1 = flen;
        }

        for (j = fidx0; j < fidx1; ++j) {
            dat0 = _mm_loadu_si128(
                (const __m128i *)
                ((const short *) in + off*2 + j*16));
            dat1 = _mm_loadu_si128(
                (const __m128i *)
                ((const short *) in + off*2 + j*16+8));
            fir0 = fd[(fn+0)*flen + j];
            fir1 = fd[(fn+1)*flen + j];

            fir0 = _mm_packs_epi32(
                _mm_srai_epi32(
                    _mm_madd_epi16(
                        _mm_unpacklo_epi16(fir0, fir1),
                        fir_interp),
                    INTERP_BITS),
                _mm_srai_epi32(
                    _mm_madd_epi16(
                        _mm_unpackhi_epi16(fir0, fir1),
                        fir_interp),
                    INTERP_BITS));

            fir1 = _mm_unpackhi_epi16(fir0, fir0);
            fir0 = _mm_unpacklo_epi16(fir0, fir0);
            acc = _mm_add_epi32(
                acc,
                _mm_add_epi32(
                    _mm_madd_epi16(
                        _mm_unpacklo_epi16(dat0, dat1),
                        _mm_unpacklo_epi16(fir0, fir1)),
                    _mm_madd_epi16(
                        _mm_unpackhi_epi16(dat0, dat1),
                        _mm_unpackhi_epi16(fir0, fir1))));
        }

    flenv_accumulate:
        switch (i & 3) {
        case 0: case 2:
            acc0 = acc;
            break;

        case 1:
            acc1 = _mm_add_epi32(
                _mm_unpacklo_epi64(acc0, acc),
                _mm_unpackhi_epi64(acc0, acc));
            break;

        case 3:
            acc0 = _mm_add_epi32(
                _mm_unpacklo_epi64(acc0, acc),
                _mm_unpackhi_epi64(acc0, acc));

            /* Apply dither */
            acc1 = _mm_add_epi32(acc1, _mm_srli_epi32(dsv, 17));
            dsv = lfr_rand_epu32(dsv, lcg_a, lcg_c);
            acc0 = _mm_add_epi32(acc0, _mm_srli_epi32(dsv, 17));
            dsv = lfr_rand_epu32(dsv, lcg_a, lcg_c);

            acc = _mm_packs_epi32(
                _mm_srai_epi32(acc1, 15),
                _mm_srai_epi32(acc0, 15));
            _mm_storeu_si128((__m128i *) ((short *) out + (i - 3) * 2), acc);
            break;
        }

        x += inv_ratio;
    }
    goto final;

flen2:
    for (i = 0; i < outlen; ++i) {
        /* fn: filter number
           ff0: filter factor for filter fn
           ff1: filter factor for filter fn+1 */
        fn = (((unsigned) x >> 1) >> (31 - log2nfilt)) &
            ((1u << log2nfilt) - 1);
        ff1 = ((unsigned) x >> (32 - log2nfilt - INTERP_BITS)) &
            ((1u << INTERP_BITS) - 1);
        ff0 = (1u << INTERP_BITS) - ff1;
        fir_interp = _mm_set1_epi32(ff0 | (ff1 << 16));

        /* off: offset in input corresponding to first sample in
           filter */
        off = (int) (x >> 32);
        /* fixd0, fidx1: start, end indexes of 8-word (16-byte) chunks
           of whole FIR data we will use */
        fidx0 = (-off + 7) >> 3;
        fidx1 = (inlen - off) >> 3;
        if (fidx0 > 0)
            goto flen2_slow;
        if (fidx1 < 2)
            goto flen2_slow;

        dat0 = _mm_loadu_si128(
            (const __m128i *)
            ((const short *) in + off*2 + 0));
        dat1 = _mm_loadu_si128(
            (const __m128i *)
            ((const short *) in + off*2 + 8));
        fir0 = fd[(fn+0)*2 + 0];
        fir1 = fd[(fn+1)*2 + 0];

        fir0 = _mm_packs_epi32(
            _mm_srai_epi32(
                _mm_madd_epi16(
                    _mm_unpacklo_epi16(fir0, fir1),
                    fir_interp),
                INTERP_BITS),
            _mm_srai_epi32(
                _mm_madd_epi16(
                    _mm_unpackhi_epi16(fir0, fir1),
                    fir_interp),
                INTERP_BITS));

        fir1 = _mm_unpackhi_epi16(fir0, fir0);
        fir0 = _mm_unpacklo_epi16(fir0, fir0);
        acc_a = _mm_madd_epi16(
            _mm_unpacklo_epi16(dat0, dat1),
            _mm_unpacklo_epi16(fir0, fir1));
        acc_b = _mm_madd_epi16(
            _mm_unpackhi_epi16(dat0, dat1),
            _mm_unpackhi_epi16(fir0, fir1));

        dat0 = _mm_loadu_si128(
            (const __m128i *)
            ((const short *) in + off*2 + 16));
        dat1 = _mm_loadu_si128(
            (const __m128i *)
            ((const short *) in + off*2 + 24));
        fir0 = fd[(fn+0)*2 + 1];
        fir1 = fd[(fn+1)*2 + 1];

        fir0 = _mm_packs_epi32(
            _mm_srai_epi32(
                _mm_madd_epi16(
                    _mm_unpacklo_epi16(fir0, fir1),
                    fir_interp),
                INTERP_BITS),
            _mm_srai_epi32(
                _mm_madd_epi16(
                    _mm_unpackhi_epi16(fir0, fir1),
                    fir_interp),
                INTERP_BITS));

        fir1 = _mm_unpackhi_epi16(fir0, fir0);
        fir0 = _mm_unpacklo_epi16(fir0, fir0);
        acc_a = _mm_add_epi32(
            acc_a,
            _mm_madd_epi16(
                _mm_unpacklo_epi16(dat0, dat1),
                _mm_unpacklo_epi16(fir0, fir1)));
        acc_b = _mm_add_epi32(
            acc_b,
            _mm_madd_epi16(
                _mm_unpackhi_epi16(dat0, dat1),
                _mm_unpackhi_epi16(fir0, fir1)));

        acc = _mm_add_epi32(acc_a, acc_b);

        goto flen2_accumulate;

    flen2_slow:
        fidx0 = -off;
        if (fidx0 < 0)
            fidx0 = 0;
        fidx1 = inlen - off;
        if (fidx1 > 16)
            fidx1 = 16;

        accs0 = 0;
        accs1 = 0;
        for (j = fidx0; j < fidx1; ++j) {
            f = (((const short *) fd)[(fn+0) * 16 + j] * ff0 +
                 ((const short *) fd)[(fn+1) * 16 + j] * ff1)
                >> INTERP_BITS;
            accs0 += ((const short *) in)[(j + off)*2 + 0] * f;
            accs1 += ((const short *) in)[(j + off)*2 + 1] * f;
        }
        acc = _mm_set_epi32(0, 0, accs1, accs0);
        goto flen2_accumulate;


    flen2_accumulate:
        switch (i & 3) {
        case 0: case 2:
            acc0 = acc;
            break;

        case 1:
            acc1 = _mm_add_epi32(
                _mm_unpacklo_epi64(acc0, acc),
                _mm_unpackhi_epi64(acc0, acc));
            break;

        case 3:
            acc0 = _mm_add_epi32(
                _mm_unpacklo_epi64(acc0, acc),
                _mm_unpackhi_epi64(acc0, acc));

            /* Apply dither */
            acc1 = _mm_add_epi32(acc1, _mm_srli_epi32(dsv, 17));
            dsv = lfr_rand_epu32(dsv, lcg_a, lcg_c);
            acc0 = _mm_add_epi32(acc0, _mm_srli_epi32(dsv, 17));
            dsv = lfr_rand_epu32(dsv, lcg_a, lcg_c);

            acc = _mm_packs_epi32(
                _mm_srai_epi32(acc1, 15),
                _mm_srai_epi32(acc0, 15));
            _mm_storeu_si128((__m128i *) ((short *) out + (i - 3) * 2), acc);
            break;
        }

        x += inv_ratio;
    }
    goto final;

flen3:
    for (i = 0; i < outlen; ++i) {
        /* fn: filter number
           ff0: filter factor for filter fn
           ff1: filter factor for filter fn+1 */
        fn = (((unsigned) x >> 1) >> (31 - log2nfilt)) &
            ((1u << log2nfilt) - 1);
        ff1 = ((unsigned) x >> (32 - log2nfilt - INTERP_BITS)) &
            ((1u << INTERP_BITS) - 1);
        ff0 = (1u << INTERP_BITS) - ff1;
        fir_interp = _mm_set1_epi32(ff0 | (ff1 << 16));

        /* acc: FIR accumulator, used for accumulating 32-bit values
           in the format L R L R.  This corresponds to one frame of
           output, so the pair of values for left and the pair for
           right have to be summed later.  */
        acc = _mm_set1_epi32(0);
        /* off: offset in input corresponding to first sample in
           filter */
        off = (int) (x >> 32);
        /* fixd0, fidx1: start, end indexes of 8-word (16-byte) chunks
           of whole FIR data we will use */
        fidx0 = (-off + 7) >> 3;
        fidx1 = (inlen - off) >> 3;
        if (fidx0 > 0)
            goto flen3_slow;
        if (fidx1 < 3)
            goto flen3_slow;

        dat0 = _mm_loadu_si128(
            (const __m128i *)
            ((const short *) in + off*2 + 0));
        dat1 = _mm_loadu_si128(
            (const __m128i *)
            ((const short *) in + off*2 + 8));
        fir0 = fd[(fn+0)*3 + 0];
        fir1 = fd[(fn+1)*3 + 0];

        fir0 = _mm_packs_epi32(
            _mm_srai_epi32(
                _mm_madd_epi16(
                    _mm_unpacklo_epi16(fir0, fir1),
                    fir_interp),
                INTERP_BITS),
            _mm_srai_epi32(
                _mm_madd_epi16(
                    _mm_unpackhi_epi16(fir0, fir1),
                    fir_interp),
                INTERP_BITS));

        fir1 = _mm_unpackhi_epi16(fir0, fir0);
        fir0 = _mm_unpacklo_epi16(fir0, fir0);
        acc_a = _mm_madd_epi16(
            _mm_unpacklo_epi16(dat0, dat1),
            _mm_unpacklo_epi16(fir0, fir1));
        acc_b = _mm_madd_epi16(
            _mm_unpackhi_epi16(dat0, dat1),
            _mm_unpackhi_epi16(fir0, fir1));

        dat0 = _mm_loadu_si128(
            (const __m128i *)
            ((const short *) in + off*2 + 16));
        dat1 = _mm_loadu_si128(
            (const __m128i *)
            ((const short *) in + off*2 + 24));
        fir0 = fd[(fn+0)*3 + 1];
        fir1 = fd[(fn+1)*3 + 1];

        fir0 = _mm_packs_epi32(
            _mm_srai_epi32(
                _mm_madd_epi16(
                    _mm_unpacklo_epi16(fir0, fir1),
                    fir_interp),
                INTERP_BITS),
            _mm_srai_epi32(
                _mm_madd_epi16(
                    _mm_unpackhi_epi16(fir0, fir1),
                    fir_interp),
                INTERP_BITS));

        fir1 = _mm_unpackhi_epi16(fir0, fir0);
        fir0 = _mm_unpacklo_epi16(fir0, fir0);
        acc_a = _mm_add_epi32(
            acc_a,
            _mm_madd_epi16(
                _mm_unpacklo_epi16(dat0, dat1),
                _mm_unpacklo_epi16(fir0, fir1)));
        acc_b = _mm_add_epi32(
            acc_b,
            _mm_madd_epi16(
                _mm_unpackhi_epi16(dat0, dat1),
                _mm_unpackhi_epi16(fir0, fir1)));

        dat0 = _mm_loadu_si128(
            (const __m128i *)
            ((const short *) in + off*2 + 32));
        dat1 = _mm_loadu_si128(
            (const __m128i *)
            ((const short *) in + off*2 + 40));
        fir0 = fd[(fn+0)*3 + 2];
        fir1 = fd[(fn+1)*3 + 2];

        fir0 = _mm_packs_epi32(
            _mm_srai_epi32(
                _mm_madd_epi16(
                    _mm_unpacklo_epi16(fir0, fir1),
                    fir_interp),
                INTERP_BITS),
            _mm_srai_epi32(
                _mm_madd_epi16(
                    _mm_unpackhi_epi16(fir0, fir1),
                    fir_interp),
                INTERP_BITS));

        fir1 = _mm_unpackhi_epi16(fir0, fir0);
        fir0 = _mm_unpacklo_epi16(fir0, fir0);
        acc_a = _mm_add_epi32(
            acc_a,
            _mm_madd_epi16(
                _mm_unpacklo_epi16(dat0, dat1),
                _mm_unpacklo_epi16(fir0, fir1)));
        acc_b = _mm_add_epi32(
            acc_b,
            _mm_madd_epi16(
                _mm_unpackhi_epi16(dat0, dat1),
                _mm_unpackhi_epi16(fir0, fir1)));

        acc = _mm_add_epi32(acc_a, acc_b);

        goto flen3_accumulate;

    flen3_slow:
        fidx0 = -off;
        if (fidx0 < 0)
            fidx0 = 0;
        fidx1 = inlen - off;
        if (fidx1 > 24)
            fidx1 = 24;

        accs0 = 0;
        accs1 = 0;
        for (j = fidx0; j < fidx1; ++j) {
            f = (((const short *) fd)[(fn+0) * 24 + j] * ff0 +
                 ((const short *) fd)[(fn+1) * 24 + j] * ff1)
                >> INTERP_BITS;
            accs0 += ((const short *) in)[(j + off)*2 + 0] * f;
            accs1 += ((const short *) in)[(j + off)*2 + 1] * f;
        }
        acc = _mm_set_epi32(0, 0, accs1, accs0);
        goto flen3_accumulate;


    flen3_accumulate:
        switch (i & 3) {
        case 0: case 2:
            acc0 = acc;
            break;

        case 1:
            acc1 = _mm_add_epi32(
                _mm_unpacklo_epi64(acc0, acc),
                _mm_unpackhi_epi64(acc0, acc));
            break;

        case 3:
            acc0 = _mm_add_epi32(
                _mm_unpacklo_epi64(acc0, acc),
                _mm_unpackhi_epi64(acc0, acc));

            /* Apply dither */
            acc1 = _mm_add_epi32(acc1, _mm_srli_epi32(dsv, 17));
            dsv = lfr_rand_epu32(dsv, lcg_a, lcg_c);
            acc0 = _mm_add_epi32(acc0, _mm_srli_epi32(dsv, 17));
            dsv = lfr_rand_epu32(dsv, lcg_a, lcg_c);

            acc = _mm_packs_epi32(
                _mm_srai_epi32(acc1, 15),
                _mm_srai_epi32(acc0, 15));
            _mm_storeu_si128((__m128i *) ((short *) out + (i - 3) * 2), acc);
            break;
        }

        x += inv_ratio;
    }
    goto final;

final:
    ds = _mm_cvtsi128_si32(dsv);
    for (i = 0; i < (outlen & 3) * 2; ++i)
        ds = LCG_A * ds + LCG_C;
    *pos = x;
    *dither = ds;

    /* Store remaing bytes */
    acc = _mm_set1_epi32(0);
    if ((i & 3) == 0)
        return;
    for (i = outlen; ; ++i) {
        switch (i & 3) {
        case 0: case 2:
            acc0 = acc;
            break;

        case 1:
            acc1 = _mm_add_epi32(
                _mm_unpacklo_epi64(acc0, acc),
                _mm_unpackhi_epi64(acc0, acc));
            break;

        case 3:
{'fraction_non_alphanumeric': 0.12026631610194648, 'fraction_numerical': 0.060048344005767355, 'mean_word_length': 1.4095228364156533, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 89, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1754980', 'n_tokens_mistral': 9768, 'n_tokens_neox': 8684, 'n_words': 1919}