<commit_before>/**
 * This is code is released under the
 * Apache License Version 2.0 http://www.apache.org/licenses/.
 */
/**
 *
 * Implementation of varint-G8IU taken from
 * Stepanov et al., SIMD-Based Decoding of Posting Lists, CIKM 2011
 * 
 * Update: <NAME> believes that this scheme was patented by Rose, Stepanov et al. (patent 20120221539).
 * We wrote this code before the patent was published (August 2012). 
 *
 * By <NAME>
 * From
 * https://github.com/maximecaron/SIMD-Based-Posting-lists
 * with minor modifications by <NAME>.
 */
#ifndef __SSSE3__
#pragma message "Disabling varintg8iu due to lack of SSSE3 support, try adding -mssse3"
#else
#ifndef VARINTG8IU_H__
#define VARINTG8IU_H__
#include <emmintrin.h>
#include "codecs.h"
#ifdef __GNUC__
#define PREDICT_FALSE(x) (__builtin_expect(x, 0))
#define PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
#else
#define PREDICT_FALSE(x) x
#define PREDICT_TRUE(x) x
#endif
typedef char v16qi __attribute__ ((vector_size (16)));

class VarIntG8IU: public IntegerCODEC {

public:

    // For all possible values of the
    // descriptor we build a table of any shuffle sequence
    // that might be needed at decode time.
    VarIntG8IU() {
        for (int desc = 0; desc <= 255; desc++) {
            int bitmask = 0x00000001;
            int bitindex = 0;
            // count number of 0 in the char
            int complete = 0;
            int ithSize[8];
            int lastpos = -1;
            while (bitindex < 8) {
                if ((desc & bitmask) == 0) {
                    ithSize[complete] = bitindex - lastpos;
                    lastpos = bitindex;
                    complete++;
                }
                bitindex++;
                bitmask = bitmask << 1;
            }
            maskOutputSize[desc] = complete;

            int j = 0;
            int k = 0;
            for (int i = 0; i < complete; i++) {
                for (int n = 0; n < 4; n++) {
                    if (n < ithSize[i]) {
                        mask[desc][k] = j;
                        j = j + 1;
                    } else {
                        mask[desc][k] = -1;
                    }
                    k = k + 1;
                }
            }

        }

    }

    void encodeArray(const uint32_t *in, const size_t length,
            uint32_t *out, size_t &nvalue) {
        const uint32_t * src = in;
        size_t srclength = length * 4;

        unsigned char* dst = reinterpret_cast<unsigned char*> (out);
        nvalue = nvalue * 4;

        size_t compressed_size = 0;
        while (srclength > 0 && nvalue >= 9) {
            compressed_size += encodeBlock(src, srclength, dst, nvalue);
        }
        //Ouput might not be a multiple of 4 so we make it so
        nvalue = (compressed_size / 4) + (compressed_size % 4 && 1);
    }<commit_msg>Fixed an odd bit of code<commit_after>/**
 * This is code is released under the
 * Apache License Version 2.0 http://www.apache.org/licenses/.
 */
/**
 *
 * Implementation of varint-G8IU taken from
 * Stepanov et al., SIMD-Based Decoding of Posting Lists, CIKM 2011
 * 
 * Update: <NAME> believes that this scheme was patented by Rose, Stepanov et al. (patent 20120221539).
 * We wrote this code before the patent was published (August 2012). 
 *
 * By <NAME>
 * From
 * https://github.com/maximecaron/SIMD-Based-Posting-lists
 * with minor modifications by <NAME>.
 */
#ifndef __SSSE3__
#pragma message "Disabling varintg8iu due to lack of SSSE3 support, try adding -mssse3"
#else
#ifndef VARINTG8IU_H__
#define VARINTG8IU_H__
#include <emmintrin.h>
#include "codecs.h"
#ifdef __GNUC__
#define PREDICT_FALSE(x) (__builtin_expect(x, 0))
#define PREDICT_TRUE(x) (__builtin_expect(!!(x), 1))
#else
#define PREDICT_FALSE(x) x
#define PREDICT_TRUE(x) x
#endif
typedef char v16qi __attribute__ ((vector_size (16)));

class VarIntG8IU: public IntegerCODEC {

public:

    // For all possible values of the
    // descriptor we build a table of any shuffle sequence
    // that might be needed at decode time.
    VarIntG8IU() {
        for (int desc = 0; desc <= 255; desc++) {
            int bitmask = 0x00000001;
            int bitindex = 0;
            // count number of 0 in the char
            int complete = 0;
            int ithSize[8];
            int lastpos = -1;
            while (bitindex < 8) {
                if ((desc & bitmask) == 0) {
                    ithSize[complete] = bitindex - lastpos;
                    lastpos = bitindex;
                    complete++;
                }
                bitindex++;
                bitmask = bitmask << 1;
            }
            maskOutputSize[desc] = complete;

            int j = 0;
            int k = 0;
            for (int i = 0; i < complete; i++) {
                for (int n = 0; n < 4; n++) {
                    if (n < ithSize[i]) {
                        mask[desc][k] = j;
                        j = j + 1;
                    } else {
                        mask[desc][k] = -1;
                    }
                    k = k + 1;
                }
            }

        }

    }

    void encodeArray(const uint32_t *in, const size_t length,
            uint32_t *out, size_t &nvalue) {
        const uint32_t * src = in;
        size_t srclength = length * 4;

        unsigned char* dst = reinterpret_cast<unsigned char*> (out);
        nvalue = nvalue * 4;

        size_t compressed_size = 0;
        while (srclength > 0 && nvalue >= 9) {
            compressed_size += encodeBlock(src, srclength, dst, nvalue);
        }
        //Ouput might not be a multiple of 4 so we make it so
        nvalue = ((compressed_size + 3 )/ 4);
    }
{'fraction_non_alphanumeric': 0.10634920634920635, 'fraction_numerical': 0.028042328042328042, 'mean_word_length': 1.723823246878002, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '768544', 'n_tokens_mistral': 1925, 'n_tokens_neox': 1718, 'n_words': 563}