<commit_before>/** \file
 *  
 *  Copyright (c) 2012-2014 by <NAME>. All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify it under the terms of the Apache License
 *  as published by the Apache Software Foundation, either version 2 of the License, or (at your option) any later
 *  version.
 *
 *  \author <NAME> (<EMAIL>)
**/
#include "char_convert.hpp"

#include "detail/fixed_map.hpp"

#include <algorithm>
#include <cassert>
#include <cctype>
#include <cstdint>
#include <iomanip>
#include <map>
#include <ostream>
#include <sstream>
#include <stdexcept>

namespace jsonv
{
namespace detail
{

decode_error::decode_error(size_type offset, const std::string& message):
        runtime_error(message),
        _offset(offset)
{ }

decode_error::~decode_error() noexcept
{ }

#define ESCAPES_LIST(item) \
    item('\b', 'b')  \
    item('\f', 'f')  \
    item('\n', 'n')  \
    item('\r', 'r')  \
    item('\t', 't')  \
    item('\\', '\\') \
    item('/',  '/')  \
    item('\"', '\"') \

#define TUPLE_PLUS_1_GEN(a, b) +1
typedef detail::fixed_map<char, char, ESCAPES_LIST(TUPLE_PLUS_1_GEN)> converter_map;

/** These entries are sorted by the numeric value of the ASCII character (\c less_entry_cpp).
 *  
 *  \note
 *  The encode and decode map must be in a different order (even though they contain the same data) because the ASCII
 *  representations of escape sequences are not in the same order as the characters they are escaping.
**/
#define TUPLE_FIRST_SECOND(a, b) { a, b },
const converter_map encode_map = { ESCAPES_LIST(TUPLE_FIRST_SECOND) };

/** These entries are sorted by the character value of the escape sequence (\c less_entry_json).
**/
#define TUPLE_SECOND_FIRST(a, b) { b, a },
const converter_map decode_map = { ESCAPES_LIST(TUPLE_SECOND_FIRST) };

const char* find(const converter_map& source, char key)
{
    converter_map::const_iterator iter = source.find(key);
    if (iter != source.end())
        return &iter->second;
    else
        return NULL;
}

static const char* find_encoding(char native_character)
{
    return find(encode_map, native_character);
}

static const char* find_decoding(char char_after_backslash)
{
    return find(decode_map, char_after_backslash);
}

static bool needs_unicode_escaping(char c)
{
    return !isprint(c)
        || (c & '\x80');
}

static constexpr bool char_bitmatch(char c, char pos, char neg)
{
    return (c & pos) == pos
        && !(c & neg);
}

/** Tests if \a c is a valid UTF-8 sequence continuation. **/
static bool is_utf8_sequence_continuation(char c)
{
    return char_bitmatch(c, '\x80', '\x40');
}

template <typename FOnError>
static void utf8_extract_info(char c, unsigned& length, char& bitmask, const FOnError& on_error)
{
    if (!(c & '\x80'))
    {
        length = 1;
        bitmask = '\x7f';
    }<commit_msg>MSVC: Workaround MS's STUPID decision to have std::isprint ASSERT if the highest order bit is set.<commit_after>/** \file
 *  
 *  Copyright (c) 2012-2014 by <NAME>. All rights reserved.
 *
 *  This program is free software: you can redistribute it and/or modify it under the terms of the Apache License
 *  as published by the Apache Software Foundation, either version 2 of the License, or (at your option) any later
 *  version.
 *
 *  \author <NAME> (<EMAIL>)
**/
#include "char_convert.hpp"

#include "detail/fixed_map.hpp"

#include <algorithm>
#include <cassert>
#include <cctype>
#include <cstdint>
#include <iomanip>
#include <map>
#include <ostream>
#include <sstream>
#include <stdexcept>

namespace jsonv
{
namespace detail
{

decode_error::decode_error(size_type offset, const std::string& message):
        runtime_error(message),
        _offset(offset)
{ }

decode_error::~decode_error() noexcept
{ }

#define ESCAPES_LIST(item) \
    item('\b', 'b')  \
    item('\f', 'f')  \
    item('\n', 'n')  \
    item('\r', 'r')  \
    item('\t', 't')  \
    item('\\', '\\') \
    item('/',  '/')  \
    item('\"', '\"') \

#define TUPLE_PLUS_1_GEN(a, b) +1
typedef detail::fixed_map<char, char, ESCAPES_LIST(TUPLE_PLUS_1_GEN)> converter_map;

/** These entries are sorted by the numeric value of the ASCII character (\c less_entry_cpp).
 *  
 *  \note
 *  The encode and decode map must be in a different order (even though they contain the same data) because the ASCII
 *  representations of escape sequences are not in the same order as the characters they are escaping.
**/
#define TUPLE_FIRST_SECOND(a, b) { a, b },
const converter_map encode_map = { ESCAPES_LIST(TUPLE_FIRST_SECOND) };

/** These entries are sorted by the character value of the escape sequence (\c less_entry_json).
**/
#define TUPLE_SECOND_FIRST(a, b) { b, a },
const converter_map decode_map = { ESCAPES_LIST(TUPLE_SECOND_FIRST) };

const char* find(const converter_map& source, char key)
{
    converter_map::const_iterator iter = source.find(key);
    if (iter != source.end())
        return &iter->second;
    else
        return NULL;
}

static const char* find_encoding(char native_character)
{
    return find(encode_map, native_character);
}

static const char* find_decoding(char char_after_backslash)
{
    return find(decode_map, char_after_backslash);
}

static bool needs_unicode_escaping(char c)
{
    #ifdef _MSC_VER
    // MSVC decided that isprint should ASSERT if the input character has the highest-order bit set, which is really
    // stupid given how sign extension works (because `char` is signed). To workaround this, we check it here because
    // this is NOT an error condition in any way, shape or form in UTF-8. That said, I do not think Microsoft ever uses
    // UTF-8.
    if (c & '\x80')
        return true;
    #endif

    return !isprint(c)
        || (c & '\x80');
}

static constexpr bool char_bitmatch(char c, char pos, char neg)
{
    return (c & pos) == pos
        && !(c & neg);
}

/** Tests if \a c is a valid UTF-8 sequence continuation. **/
static bool is_utf8_sequence_continuation(char c)
{
    return char_bitmatch(c, '\x80', '\x40');
}

template <typename FOnError>
static void utf8_extract_info(char c, unsigned& length, char& bitmask, const FOnError& on_error)
{
    if (!(c & '\x80'))
    {
        length = 1;
        bitmask = '\x7f';
    }
{'fraction_non_alphanumeric': 0.12903225806451613, 'fraction_numerical': 0.008666345690900338, 'mean_word_length': 3.5958702064896757, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6857229', 'n_tokens_mistral': 2147, 'n_tokens_neox': 2076, 'n_words': 665}