<commit_before>#ifndef JSON_SPIRIT_READER_TEMPLATE
#define JSON_SPIRIT_READER_TEMPLATE

//          Copyright <NAME> 2007 - 2011
// Distributed under the MIT License, see accompanying file LICENSE.txt

// json spirit version 4.05

#if defined(_MSC_VER) && (_MSC_VER >= 1020)
# pragma once
#endif

#include "value.h"
#include "parse_error.h"

#define BOOST_SPIRIT_THREADSAFE  // uncomment for multithreaded use, requires linking to boost.thread

#include <boost/bind.hpp>
#include <boost/function.hpp>
#include <boost/version.hpp>

#if BOOST_VERSION >= 103800
    #include <boost/spirit/include/classic_core.hpp>
    #include <boost/spirit/include/classic_confix.hpp>
    #include <boost/spirit/include/classic_escape_char.hpp>
    #include <boost/spirit/include/classic_multi_pass.hpp>
    #include <boost/spirit/include/classic_position_iterator.hpp>
    #define spirit_namespace boost::spirit::classic
#else
    #include <boost/spirit/core.hpp>
    #include <boost/spirit/utility/confix.hpp>
    #include <boost/spirit/utility/escape_char.hpp>
    #include <boost/spirit/iterator/multi_pass.hpp>
    #include <boost/spirit/iterator/position_iterator.hpp>
    #define spirit_namespace boost::spirit
#endif

namespace json_spirit
{
    const spirit_namespace::int_parser < boost::int64_t >  int64_p  = spirit_namespace::int_parser < boost::int64_t  >();
    const spirit_namespace::uint_parser< boost::uint64_t > uint64_p = spirit_namespace::uint_parser< boost::uint64_t >();

    template< class Iter_type >
    bool is_eq( Iter_type first, Iter_type last, const char* c_str )
    {
        for( Iter_type i = first; i != last; ++i, ++c_str )
        {
            if( *c_str == 0 ) return false;

            if( *i != *c_str ) return false;
        }

        return true;
    }

    template< class Char_type >
    Char_type hex_to_num( const Char_type c )
    {
        if( ( c >= '0' ) && ( c <= '9' ) ) return c - '0';
        if( ( c >= 'a' ) && ( c <= 'f' ) ) return c - 'a' + 10;
        if( ( c >= 'A' ) && ( c <= 'F' ) ) return c - 'A' + 10;
        return 0;
    }

    template< class Char_type, class Iter_type >
    Char_type hex_str_to_char( Iter_type& begin )
    {
        const Char_type c1( *( ++begin ) );
        const Char_type c2( *( ++begin ) );

        return ( hex_to_num( c1 ) << 4 ) + hex_to_num( c2 );
    }

    template< class Char_type, class Iter_type >
    std::basic_string<Char_type> unicode_str_to_char( Iter_type& begin )
    {
        const Char_type c1( *( ++begin ) );
        const Char_type c2( *( ++begin ) );
        const Char_type c3( *( ++begin ) );
        const Char_type c4( *( ++begin ) );
        Char_type res[2];
        res[0] = ( hex_to_num( c1 ) << 12 ) +
                 ( hex_to_num( c2 ) <<  8 ) +
                 ( hex_to_num( c3 ) <<  4 ) +
                 hex_to_num( c4 );
        res[1] = 0;
        return res;
    }

    // Samelessly stolen from the internet:
    // https://stackoverflow.com/questions/4607413/c-library-to-convert-unicode-code-points-to-utf8
    static
    std::string
    codepoint_to_utf8(int c)
    {
      char res[5] = {0, 0, 0, 0, 0};
      if (c < 0x80)
      {
        res[0] = c;
      }
      else if (c < 0x800)
      {
        res[0] = 192 + c / 64;
        res[1] = 128 + c % 64;
      }
      else if (c - 0xd800u < 0x800)
        throw std::runtime_error("invalid unicode codepoint: " + std::to_string(c));
      else if (c<0x10000)
      {
        res[0] = 224 + c / 4096;
        res[1] = 128 + c / 64 % 64;
        res[2] = 128 + c % 64;
      }<commit_msg>Do not throw on surrogate pairs, output '?'<commit_after>#ifndef JSON_SPIRIT_READER_TEMPLATE
#define JSON_SPIRIT_READER_TEMPLATE

//          Copyright <NAME> 2007 - 2011
// Distributed under the MIT License, see accompanying file LICENSE.txt

// json spirit version 4.05

#if defined(_MSC_VER) && (_MSC_VER >= 1020)
# pragma once
#endif

#include "value.h"
#include "parse_error.h"

#define BOOST_SPIRIT_THREADSAFE  // uncomment for multithreaded use, requires linking to boost.thread

#include <boost/bind.hpp>
#include <boost/function.hpp>
#include <boost/version.hpp>

#if BOOST_VERSION >= 103800
    #include <boost/spirit/include/classic_core.hpp>
    #include <boost/spirit/include/classic_confix.hpp>
    #include <boost/spirit/include/classic_escape_char.hpp>
    #include <boost/spirit/include/classic_multi_pass.hpp>
    #include <boost/spirit/include/classic_position_iterator.hpp>
    #define spirit_namespace boost::spirit::classic
#else
    #include <boost/spirit/core.hpp>
    #include <boost/spirit/utility/confix.hpp>
    #include <boost/spirit/utility/escape_char.hpp>
    #include <boost/spirit/iterator/multi_pass.hpp>
    #include <boost/spirit/iterator/position_iterator.hpp>
    #define spirit_namespace boost::spirit
#endif

namespace json_spirit
{
    const spirit_namespace::int_parser < boost::int64_t >  int64_p  = spirit_namespace::int_parser < boost::int64_t  >();
    const spirit_namespace::uint_parser< boost::uint64_t > uint64_p = spirit_namespace::uint_parser< boost::uint64_t >();

    template< class Iter_type >
    bool is_eq( Iter_type first, Iter_type last, const char* c_str )
    {
        for( Iter_type i = first; i != last; ++i, ++c_str )
        {
            if( *c_str == 0 ) return false;

            if( *i != *c_str ) return false;
        }

        return true;
    }

    template< class Char_type >
    Char_type hex_to_num( const Char_type c )
    {
        if( ( c >= '0' ) && ( c <= '9' ) ) return c - '0';
        if( ( c >= 'a' ) && ( c <= 'f' ) ) return c - 'a' + 10;
        if( ( c >= 'A' ) && ( c <= 'F' ) ) return c - 'A' + 10;
        return 0;
    }

    template< class Char_type, class Iter_type >
    Char_type hex_str_to_char( Iter_type& begin )
    {
        const Char_type c1( *( ++begin ) );
        const Char_type c2( *( ++begin ) );

        return ( hex_to_num( c1 ) << 4 ) + hex_to_num( c2 );
    }

    template< class Char_type, class Iter_type >
    std::basic_string<Char_type> unicode_str_to_char( Iter_type& begin )
    {
        const Char_type c1( *( ++begin ) );
        const Char_type c2( *( ++begin ) );
        const Char_type c3( *( ++begin ) );
        const Char_type c4( *( ++begin ) );
        Char_type res[2];
        res[0] = ( hex_to_num( c1 ) << 12 ) +
                 ( hex_to_num( c2 ) <<  8 ) +
                 ( hex_to_num( c3 ) <<  4 ) +
                 hex_to_num( c4 );
        res[1] = 0;
        return res;
    }

    // Samelessly stolen from the internet:
    // https://stackoverflow.com/questions/4607413/c-library-to-convert-unicode-code-points-to-utf8
    static
    std::string
    codepoint_to_utf8(int c)
    {
      char res[5] = {0, 0, 0, 0, 0};
      if (c < 0x80)
      {
        res[0] = c;
      }
      else if (c < 0x800)
      {
        res[0] = 192 + c / 64;
        res[1] = 128 + c % 64;
      }
      else if (c - 0xd800u < 0x800)
      {
        //FIXME: this is probably a character from a surrogate pair ie codepoint above 0xFFFF
        // that must be encoded in two characters because the json specs says \u is followed
        // by exactly 4 hex digits.
        res[0] = '?';
      }
      else if (c<0x10000)
      {
        res[0] = 224 + c / 4096;
        res[1] = 128 + c / 64 % 64;
        res[2] = 128 + c % 64;
      }
{'fraction_non_alphanumeric': 0.13524925088531736, 'fraction_numerical': 0.0369109234540997, 'mean_word_length': 2.386992619926199, 'pattern_counts': {'":': 0, '<': 79, '<?xml version=': 0, '>': 59, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6328746', 'n_tokens_mistral': 2785, 'n_tokens_neox': 2548, 'n_words': 611}