<commit_before>#ifndef VTRC_SIZEPACK_POLICY_H
#define VTRC_SIZEPACK_POLICY_H

#include <string>

#include <limits>
#include <boost/static_assert.hpp>

/*
size_packer_policy {

    type    size_type
    size_t  max_length;

    template <typename IterT>
    static size_t size_length( IterT begin, const IterT &end );

    template <typename IterT>
    static size_type unpack( IterT begin, const IterT &end )

    static size_t packed_length( size_type input )
    static std::string pack( size_type size )

}
*/

namespace vtrc { namespace common { namespace policies {

    template <typename SizeType>
    struct fixint_policy {
    private:
        BOOST_STATIC_ASSERT(
            !std::numeric_limits<SizeType>::is_signed
        );
    public:

        typedef SizeType size_type;
        static const size_t max_length = sizeof(SizeType);

        template <typename IterT>
        static size_t size_length( IterT begin, const IterT &end )
        {
            return std::distance(begin, end) >= max_length ? max_length : 0;
        }

        static size_t packed_length( size_type input )
        {
            return max_length;
        }

        static std::string pack( size_type size )
        {
            std::string res(max_length, '\0');
            for( size_t current =  max_length; current > 0; --current ) {
                res[current-1] = static_cast<char>( size & 0xFF );
                size >>= 8;
            }
            return res;
        }

        template <typename IterT>
        static size_type unpack( IterT begin, const IterT &end )
        {
            size_type  res   = 0x00;<commit_msg>size pack policy<commit_after>#ifndef VTRC_SIZEPACK_POLICY_H
#define VTRC_SIZEPACK_POLICY_H

#include <string>

#include <limits>
#include <boost/static_assert.hpp>

/*
size_packer_policy {

    type    size_type
    size_t  max_length;

    template <typename IterT>
    static size_t size_length( IterT begin, const IterT &end );

    template <typename IterT>
    static size_type unpack( IterT begin, const IterT &end )

    static size_t packed_length( size_type input )
    static std::string pack( size_type size )

}
*/

namespace vtrc { namespace common { namespace policies {

    template <typename SizeType>
    struct fixint_policy {
    private:
        BOOST_STATIC_ASSERT(
            !std::numeric_limits<SizeType>::is_signed
        );
    public:

        typedef SizeType size_type;
        static const size_t max_length = sizeof(SizeType);

        template <typename IterT>
        static size_t size_length( IterT begin, const IterT &end )
        {
            return std::distance(begin, end) >= max_length ? max_length : 0;
        }

        static size_t packed_length( size_type input )
        {
            return max_length;
        }

        static std::string pack( size_type size )
        {
            char res[max_length];
            for( size_t current =  max_length; current > 0; --current ) {
                res[current-1] = static_cast<char>( size & 0xFF );
                size >>= 8;
            }
            return std::string( &res[0], &res[max_length] );
        }

        template <typename IterT>
        static size_type unpack( IterT begin, const IterT &end )
        {
            size_type  res   = 0x00;
{'fraction_non_alphanumeric': 0.08457108214610488, 'fraction_numerical': 0.005456198848135798, 'mean_word_length': 2.216374269005848, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5089609', 'n_tokens_mistral': 1021, 'n_tokens_neox': 975, 'n_words': 264}