<commit_before>#ifndef __SERIALIZER_SEMANTIC_CHECKING_HPP__
#define __SERIALIZER_SEMANTIC_CHECKING_HPP__

#include "errors.hpp"
#include <string>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <boost/crc.hpp>
#include "arch/arch.hpp"
#include "config/args.hpp"
#include "serializer/serializer.hpp"
#include "containers/two_level_array.hpp"

/* This is a thin wrapper around the log serializer that makes sure that the
serializer has the correct semantics. It must have exactly the same API as
the log serializer. */

//#define SERIALIZER_DEBUG_PRINT 1

// TODO (rntz) fix this for new serializer interface
template<class inner_serializer_t>
class semantic_checking_serializer_t :
    public serializer_t
{
private:
    inner_serializer_t inner_serializer;

    typedef uint32_t crc_t;

    struct block_info_t {
        enum state_t {
            state_unknown,
            state_deleted,
            state_have_crc
        } state;
        crc_t crc;

        // For compatibility with two_level_array_t
        block_info_t() : state(state_unknown) {}
        operator bool() { return state != state_unknown; }
    };

    two_level_array_t<block_info_t, MAX_BLOCK_ID> blocks;

    crc_t compute_crc(const void *buf) {
        boost::crc_optimal<32, 0x04C11DB7, 0xFFFFFFFF, 0xFFFFFFFF, true, true> crc_computer;
        // We need to not crc BLOCK_META_DATA_SIZE because it's
        // internal to the serializer.
        crc_computer.process_bytes(buf, get_block_size().value());<commit_msg>serializer/semantic_checking.hpp: add TODO<commit_after>#ifndef __SERIALIZER_SEMANTIC_CHECKING_HPP__
#define __SERIALIZER_SEMANTIC_CHECKING_HPP__

#include "errors.hpp"
#include <string>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <boost/crc.hpp>
#include "arch/arch.hpp"
#include "config/args.hpp"
#include "serializer/serializer.hpp"
#include "containers/two_level_array.hpp"

/* This is a thin wrapper around the log serializer that makes sure that the
serializer has the correct semantics. It must have exactly the same API as
the log serializer. */

//#define SERIALIZER_DEBUG_PRINT 1

// TODO make this just an interface and move implementation into semantic_checking.tcc, then use
// tim's template hack to instantiate as necessary in a .cc file.

// TODO (rntz) fix this for new serializer interface
template<class inner_serializer_t>
class semantic_checking_serializer_t :
    public serializer_t
{
private:
    inner_serializer_t inner_serializer;

    typedef uint32_t crc_t;

    struct block_info_t {
        enum state_t {
            state_unknown,
            state_deleted,
            state_have_crc
        } state;
        crc_t crc;

        // For compatibility with two_level_array_t
        block_info_t() : state(state_unknown) {}
        operator bool() { return state != state_unknown; }
    };

    two_level_array_t<block_info_t, MAX_BLOCK_ID> blocks;

    crc_t compute_crc(const void *buf) {
        boost::crc_optimal<32, 0x04C11DB7, 0xFFFFFFFF, 0xFFFFFFFF, true, true> crc_computer;
        // We need to not crc BLOCK_META_DATA_SIZE because it's
        // internal to the serializer.
        crc_computer.process_bytes(buf, get_block_size().value());
{'fraction_non_alphanumeric': 0.08408037094281298, 'fraction_numerical': 0.0080370942812983, 'mean_word_length': 3.642754662840746, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 4, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7470179', 'n_tokens_mistral': 1110, 'n_tokens_neox': 1050, 'n_words': 277}