<commit_before>//****************************************************************************
// Copyright © 2015 <NAME>. All rights reserved.
// Created by <NAME> on 2015-12-05.
//
// This file is distributed under the BSD License.
// License text is included with the source distribution.
//****************************************************************************
#include "GetValueType.hpp"

#include <cstdint>

namespace Yson {

    namespace {
        int getDigit(char c);

        bool isHexDigit(char c);

        template<typename Predicate>
        ValueType_t getValueType(const char* first, const char* last,
                                 Predicate isDigit, ValueType_t type);

        ValueType_t getBinaryValueType(const char* first, const char* last);

        ValueType_t getOctalValueType(const char* first, const char* last);

        ValueType_t getHexadecimalValueType(const char* first,
                                            const char* last);

        ValueType_t getFloatingPointValueType(const char* first,
                                              const char* last);

        ValueType_t getNumberValueType(const char* first, const char* last);
    }

    ValueType_t getValueType(const char* first, const char* last)
    {
        if (first == last)
            return ValueType::INVALID;
        auto assumedType = ValueType_t(ValueType::UNKNOWN);
        if (*first == '-' || *first == '+')
        {
            if (++first == last)
                return ValueType::INVALID;
            assumedType = ValueType::INTEGER;
        }
        if (*first == '0')
        {
            if (++first == last)
                return ValueType::INTEGER;
            switch (*first)
            {
            case 'b':
                return getBinaryValueType(++first, last);
            case 'o':
                return getOctalValueType(++first, last);
            case 'x':
                return getHexadecimalValueType(++first, last);
            case '.':
            case 'e':
            case 'E':
                return getFloatingPointValueType(first, last);
            default:
                if (getDigit(*first) <= 9)
                {
                    if (++first == last)
                        return ValueType::INTEGER;
                    return getNumberValueType(first, last);
                }
                return ValueType::INVALID;
            }
        }
        else if (getDigit(*first) <= 9)
        {
            if (++first == last)
                return ValueType::INTEGER;
            return getNumberValueType(first, last);
        }

        if (last - first == 8 && std::equal(first, last, "infinity"))
            return ValueType::EXTENDED_FLOAT;<commit_msg>Recognize hex, oct and binary values with uppercase prefixes.<commit_after>//****************************************************************************
// Copyright © 2015 <NAME>. All rights reserved.
// Created by <NAME> on 2015-12-05.
//
// This file is distributed under the BSD License.
// License text is included with the source distribution.
//****************************************************************************
#include "GetValueType.hpp"

#include <cstdint>

namespace Yson {

    namespace {
        int getDigit(char c);

        bool isHexDigit(char c);

        template<typename Predicate>
        ValueType_t getValueType(const char* first, const char* last,
                                 Predicate isDigit, ValueType_t type);

        ValueType_t getBinaryValueType(const char* first, const char* last);

        ValueType_t getOctalValueType(const char* first, const char* last);

        ValueType_t getHexadecimalValueType(const char* first,
                                            const char* last);

        ValueType_t getFloatingPointValueType(const char* first,
                                              const char* last);

        ValueType_t getNumberValueType(const char* first, const char* last);
    }

    ValueType_t getValueType(const char* first, const char* last)
    {
        if (first == last)
            return ValueType::INVALID;
        auto assumedType = ValueType_t(ValueType::UNKNOWN);
        if (*first == '-' || *first == '+')
        {
            if (++first == last)
                return ValueType::INVALID;
            assumedType = ValueType::INTEGER;
        }
        if (*first == '0')
        {
            if (++first == last)
                return ValueType::INTEGER;
            switch (*first)
            {
            case 'b':
            case 'B':
                return getBinaryValueType(++first, last);
            case 'o':
            case 'O':
                return getOctalValueType(++first, last);
            case 'x':
            case 'X':
                return getHexadecimalValueType(++first, last);
            case '.':
            case 'e':
            case 'E':
                return getFloatingPointValueType(first, last);
            default:
                if (getDigit(*first) <= 9)
                {
                    if (++first == last)
                        return ValueType::INTEGER;
                    return getNumberValueType(first, last);
                }
                return ValueType::INVALID;
            }
        }
        else if (getDigit(*first) <= 9)
        {
            if (++first == last)
                return ValueType::INTEGER;
            return getNumberValueType(first, last);
        }

        if (last - first == 8 && std::equal(first, last, "infinity"))
            return ValueType::EXTENDED_FLOAT;
{'fraction_non_alphanumeric': 0.15100071479628305, 'fraction_numerical': 0.005718370264474625, 'mean_word_length': 1.703864734299517, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7208995', 'n_tokens_mistral': 1444, 'n_tokens_neox': 1351, 'n_words': 379}