<commit_before>
#ifndef __SERIALIZER_SEMANTIC_CHECKING_HPP__
#define __SERIALIZER_SEMANTIC_CHECKING_HPP__

#include <boost/crc.hpp>
#include "config/args.hpp"

/* This is a thin wrapper around the log serializer that makes sure that the
serializer has the correct semantics. It must have exactly the same API as
the log serializer. */

// #define SERIALIZER_DEBUG_PRINT 1

template<class inner_serializer_t>
class semantic_checking_serializer_t
{
public:
    typedef data_block_manager_t::buf_data_t buf_data_t;
private:
    inner_serializer_t inner_serializer;
    
    typedef uint32_t crc_t;
    
    struct block_info_t {
        
        enum state_t {
            state_unknown,
            state_deleted,
            state_have_crc
        } state;
        crc_t crc;
        
        // For compatibility with two_level_array_t
        block_info_t() : state(state_unknown) {}
        operator bool() { return state != state_unknown; }
    };
    
    two_level_array_t<block_info_t, MAX_BLOCK_ID> blocks;
    
    crc_t compute_crc(void *buf) {
        boost::crc_optimal<32, 0x04C11DB7, 0xFFFFFFFF, 0xFFFFFFFF, true, true> crc_computer;
        crc_computer.process_bytes(buf, block_size);
        return crc_computer.checksum();
    }
    
    int last_write_started, last_write_callbacked;
    
public:
    size_t block_size;
    
    semantic_checking_serializer_t(char *db_path, size_t block_size)
        : inner_serializer(db_path, block_size),
          last_write_started(0), last_write_callbacked(0),
          block_size(block_size) {}

public:
    typedef typename inner_serializer_t::ready_callback_t ready_callback_t;
    bool start(ready_callback_t *ready_cb) {
        return inner_serializer.start(ready_cb);
    }
    
public:
    /* For reads, we check to make sure that the data we get back in the read is
    consistent with what was last written there. */
    
    struct read_callback_t {
        virtual void on_serializer_read() = 0;
    };
    
    struct reader_t :<commit_msg>We need to account for BLOCK_META_DATA_SIZE in the semantic checker<commit_after>
#ifndef __SERIALIZER_SEMANTIC_CHECKING_HPP__
#define __SERIALIZER_SEMANTIC_CHECKING_HPP__

#include <boost/crc.hpp>
#include "config/args.hpp"

/* This is a thin wrapper around the log serializer that makes sure that the
serializer has the correct semantics. It must have exactly the same API as
the log serializer. */

// #define SERIALIZER_DEBUG_PRINT 1

template<class inner_serializer_t>
class semantic_checking_serializer_t
{
public:
    typedef data_block_manager_t::buf_data_t buf_data_t;
private:
    inner_serializer_t inner_serializer;
    
    typedef uint32_t crc_t;
    
    struct block_info_t {
        
        enum state_t {
            state_unknown,
            state_deleted,
            state_have_crc
        } state;
        crc_t crc;
        
        // For compatibility with two_level_array_t
        block_info_t() : state(state_unknown) {}
        operator bool() { return state != state_unknown; }
    };
    
    two_level_array_t<block_info_t, MAX_BLOCK_ID> blocks;
    
    crc_t compute_crc(void *buf) {
        boost::crc_optimal<32, 0x04C11DB7, 0xFFFFFFFF, 0xFFFFFFFF, true, true> crc_computer;
        // We need to not crc BLOCK_META_DATA_SIZE because it's
        // internal to the serializer.
        crc_computer.process_bytes((void*)((char*)buf + BLOCK_META_DATA_SIZE), block_size - BLOCK_META_DATA_SIZE);
        return crc_computer.checksum();
    }
    
    int last_write_started, last_write_callbacked;
    
public:
    size_t block_size;
    
    semantic_checking_serializer_t(char *db_path, size_t block_size)
        : inner_serializer(db_path, block_size),
          last_write_started(0), last_write_callbacked(0),
          block_size(block_size) {}

public:
    typedef typename inner_serializer_t::ready_callback_t ready_callback_t;
    bool start(ready_callback_t *ready_cb) {
        return inner_serializer.start(ready_cb);
    }
    
public:
    /* For reads, we check to make sure that the data we get back in the read is
    consistent with what was last written there. */
    
    struct read_callback_t {
        virtual void on_serializer_read() = 0;
    };
    
    struct reader_t :
{'fraction_non_alphanumeric': 0.06878680800942286, 'fraction_numerical': 0.007538280329799764, 'mean_word_length': 3.074856046065259, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '913890', 'n_tokens_mistral': 1432, 'n_tokens_neox': 1374, 'n_words': 327}