<commit_before>
#include "tdb_internal.h"
#include "huffman.h"
#include "util.h"

uint32_t tdb_decode_trail(tdb *db,
                          uint64_t cookie_id,
                          uint32_t *dst,
                          uint32_t dst_size,
                          int edge_encoded)
{
    const uint32_t *toc;
    const char *data;
    uint64_t item, size, offs = 0;
    const struct huff_codebook *codebook =
        (const struct huff_codebook*)db->codebook.data;
    uint32_t j, i = 0;
    uint32_t tstamp = db->min_timestamp;
    uint64_t delta, prev_offs;
    const struct field_stats *fstats = db->field_stats;
    tdb_field field, num_ofields = db->num_fields - 1;

    if (cookie_id >= db->num_cookies)
        return 0;

    toc = (const uint32_t*)db->trails.data;
    data = &db->trails.data[toc[cookie_id]];
    size = 8 * (toc[cookie_id + 1] - toc[cookie_id]);
    size -= read_bits(data, 0, 3);
    offs = 3;
    if (edge_encoded){
        while (offs < size && i < dst_size){
            /* Every event starts with a timestamp.
               Timestamp may be the first member of a bigram */
            item = huff_decode_value(codebook, data, &offs, fstats);
            delta = (item & UINT32_MAX) >> 8;
            if (delta == TDB_FAR_TIMEDELTA){
                dst[i++] = 0;
            }else{
                tstamp += delta;
                dst[i++] = tstamp;
            }
            item >>= 32;
            if (item && i < dst_size)
                dst[i++] = item;

            /* timestamp is followed by at most num_ofields field values */
            while (offs < size && i < dst_size){
                prev_offs = offs;
                item = huff_decode_value(codebook, data, &offs, fstats);
                field = tdb_item_field(item);
                if (field){
                    /* value may be either a unigram or a bigram */
                    do{
                        dst[i++] = item & UINT32_MAX;
                        item >>= 32;
                    }while (item && i < dst_size);
                }else{
                    /* we hit the next timestamp, take a step back and break */
                    offs = prev_offs;
                    break;
                }
            }
            /* we mark the end of an event with zero. Zero can't occur in any
               of the field values since the timestamp field is 0 */
            if (i < dst_size)
                dst[i++] = 0;
        }
    }else{
        /* same thing as above but here we decode edge encoding */
        memset(db->previous_items, 0, num_ofields * 4);

        while (offs < size && i < dst_size){
            item = huff_decode_value(codebook, data, &offs, fstats);
            delta = (item & UINT32_MAX) >> 8;
            if (delta == TDB_FAR_TIMEDELTA){
                dst[i++] = 0;
            }else{
                tstamp += delta;
                dst[i++] = tstamp;
            }
            item >>= 32;
            field = tdb_item_field(item);

            if (field)
                db->previous_items[field - 1] = item;

            /* edge encoding: some fields may be inherited from previous
               events - keep track what we have seen in the past */
            while (offs < size){
                prev_offs = offs;
                item = huff_decode_value(codebook, data, &offs, fstats);
                field = tdb_item_field(item);
                if (field){
                    do{
                        db->previous_items[field - 1] = item & UINT32_MAX;
                        item >>= 32;
                    }while ((field = tdb_item_field(item)));<commit_msg>trail decode: initialize previous items with fields<commit_after>
#include "tdb_internal.h"
#include "huffman.h"
#include "util.h"

uint32_t tdb_decode_trail(tdb *db,
                          uint64_t cookie_id,
                          uint32_t *dst,
                          uint32_t dst_size,
                          int edge_encoded)
{
    const uint32_t *toc;
    const char *data;
    uint64_t item, size, offs = 0;
    const struct huff_codebook *codebook =
        (const struct huff_codebook*)db->codebook.data;
    uint32_t k, j, i = 0;
    uint32_t tstamp = db->min_timestamp;
    uint64_t delta, prev_offs;
    const struct field_stats *fstats = db->field_stats;
    tdb_field field, num_ofields = db->num_fields - 1;

    if (cookie_id >= db->num_cookies)
        return 0;

    toc = (const uint32_t*)db->trails.data;
    data = &db->trails.data[toc[cookie_id]];
    size = 8 * (toc[cookie_id + 1] - toc[cookie_id]);
    size -= read_bits(data, 0, 3);
    offs = 3;
    if (edge_encoded){
        while (offs < size && i < dst_size){
            /* Every event starts with a timestamp.
               Timestamp may be the first member of a bigram */
            item = huff_decode_value(codebook, data, &offs, fstats);
            delta = (item & UINT32_MAX) >> 8;
            if (delta == TDB_FAR_TIMEDELTA){
                dst[i++] = 0;
            }else{
                tstamp += delta;
                dst[i++] = tstamp;
            }
            item >>= 32;
            if (item && i < dst_size)
                dst[i++] = item;

            /* timestamp is followed by at most num_ofields field values */
            while (offs < size && i < dst_size){
                prev_offs = offs;
                item = huff_decode_value(codebook, data, &offs, fstats);
                field = tdb_item_field(item);
                if (field){
                    /* value may be either a unigram or a bigram */
                    do{
                        dst[i++] = item & UINT32_MAX;
                        item >>= 32;
                    }while (item && i < dst_size);
                }else{
                    /* we hit the next timestamp, take a step back and break */
                    offs = prev_offs;
                    break;
                }
            }
            /* we mark the end of an event with zero. Zero can't occur in any
               of the field values since the timestamp field is 0 */
            if (i < dst_size)
                dst[i++] = 0;
        }
    }else{
        /* same thing as above but here we decode edge encoding */
        memset(db->previous_items, 0, num_ofields * 4);
        for (k = 0; k < num_ofields; k++)
            db->previous_items[k] = k + 1;

        while (offs < size && i < dst_size){
            item = huff_decode_value(codebook, data, &offs, fstats);
            delta = (item & UINT32_MAX) >> 8;
            if (delta == TDB_FAR_TIMEDELTA){
                dst[i++] = 0;
            }else{
                tstamp += delta;
                dst[i++] = tstamp;
            }
            item >>= 32;
            field = tdb_item_field(item);

            if (field)
                db->previous_items[field - 1] = item;

            /* edge encoding: some fields may be inherited from previous
               events - keep track what we have seen in the past */
            while (offs < size){
                prev_offs = offs;
                item = huff_decode_value(codebook, data, &offs, fstats);
                field = tdb_item_field(item);
                if (field){
                    do{
                        db->previous_items[field - 1] = item & UINT32_MAX;
                        item >>= 32;
                    }while ((field = tdb_item_field(item)));
{'fraction_non_alphanumeric': 0.10434665226781857, 'fraction_numerical': 0.01511879049676026, 'mean_word_length': 1.3595541401273885, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6153401', 'n_tokens_mistral': 2314, 'n_tokens_neox': 2137, 'n_words': 625}