<commit_before>// Internal to the library, do not include this directly

#pragma once

#include "unicorn/core.hpp"
#include "unicorn/character.hpp"
#include "unicorn/string-forward.hpp"
#include "unicorn/string-property.hpp"
#include "unicorn/string-size.hpp"
#include "unicorn/utf.hpp"
#include <algorithm>
#include <cerrno>
#include <cmath>
#include <iterator>
#include <limits>
#include <stdexcept>
#include <string>
#include <type_traits>

namespace Unicorn {

    namespace UnicornDetail {

        template <typename T, typename C>
        UtfIterator<C> convert_str_to_int(T& t, const UtfIterator<C>& start, uint32_t flags, int base) {
            PRI_STATIC_ASSERT(std::is_integral<T>::value);
            static const auto dec_chars = recode<C>(u8string("+-0123456789"));
            static const auto hex_chars = recode<C>(u8string("+-0123456789ABCDEFabcdef"));
            const auto& src(start.source());
            size_t offset = start.offset();
            if (offset >= src.size()) {
                if (flags & err_throw)
                    throw std::invalid_argument("Invalid integer (empty string)");
                t = T(0);
                return utf_end(src);
            }
            size_t endpos = src.find_first_not_of(base == 16 ? hex_chars : dec_chars, offset);
            if (endpos == offset) {
                if (flags & err_throw)
                    throw std::invalid_argument("Invalid integer: " + quote(to_utf8(start.str())));
                t = T(0);
                return start;
            }
            if (endpos == npos)
                endpos = src.size();
            u8string fragment(endpos - offset, '\0');
            std::transform(src.begin() + offset, src.begin() + endpos, fragment.begin(),
                [] (C c) { return char(c); });
            UtfIterator<C> stop;
            if (std::is_signed<T>::value) {
                static constexpr auto min_value = static_cast<long long>(std::numeric_limits<T>::min());
                static constexpr auto max_value = static_cast<long long>(std::numeric_limits<T>::max());
                char* endptr = nullptr;
                errno = 0;
                long long value = strtoll(fragment.data(), &endptr, base);
                int err = errno;
                size_t len = endptr - fragment.data();
                stop = utf_iterator(src, offset + len);
                if (len == 0) {
                    if (flags & err_throw)
                        throw std::invalid_argument("Invalid integer: " + quote(fragment));
                    t = T(0);
                } else if (err == ERANGE || value < min_value || value > max_value) {
                    if (flags & err_throw)
                        throw std::range_error("Integer out of range: " + quote(fragment));
                    t = value > T(0) ? max_value : min_value;
                } else {
                    t = value;
                }
            } else {<commit_msg>Fix GCC complaint about sign comparison<commit_after>// Internal to the library, do not include this directly

#pragma once

#include "unicorn/core.hpp"
#include "unicorn/character.hpp"
#include "unicorn/string-forward.hpp"
#include "unicorn/string-property.hpp"
#include "unicorn/string-size.hpp"
#include "unicorn/utf.hpp"
#include <algorithm>
#include <cerrno>
#include <cmath>
#include <iterator>
#include <limits>
#include <stdexcept>
#include <string>
#include <type_traits>

namespace Unicorn {

    namespace UnicornDetail {

        template <typename T, typename C>
        UtfIterator<C> convert_str_to_int(T& t, const UtfIterator<C>& start, uint32_t flags, int base) {
            PRI_STATIC_ASSERT(std::is_integral<T>::value);
            static const auto dec_chars = recode<C>(u8string("+-0123456789"));
            static const auto hex_chars = recode<C>(u8string("+-0123456789ABCDEFabcdef"));
            const auto& src(start.source());
            size_t offset = start.offset();
            if (offset >= src.size()) {
                if (flags & err_throw)
                    throw std::invalid_argument("Invalid integer (empty string)");
                t = T(0);
                return utf_end(src);
            }
            size_t endpos = src.find_first_not_of(base == 16 ? hex_chars : dec_chars, offset);
            if (endpos == offset) {
                if (flags & err_throw)
                    throw std::invalid_argument("Invalid integer: " + quote(to_utf8(start.str())));
                t = T(0);
                return start;
            }
            if (endpos == npos)
                endpos = src.size();
            u8string fragment(endpos - offset, '\0');
            std::transform(src.begin() + offset, src.begin() + endpos, fragment.begin(),
                [] (C c) { return char(c); });
            UtfIterator<C> stop;
            if (std::is_signed<T>::value) {
                static constexpr auto min_value = static_cast<long long>(std::numeric_limits<T>::min());
                static constexpr auto max_value = static_cast<long long>(std::numeric_limits<T>::max());
                char* endptr = nullptr;
                errno = 0;
                long long value = strtoll(fragment.data(), &endptr, base);
                int err = errno;
                size_t len = endptr - fragment.data();
                stop = utf_iterator(src, offset + len);
                if (len == 0) {
                    if (flags & err_throw)
                        throw std::invalid_argument("Invalid integer: " + quote(fragment));
                    t = T(0);
                } else if (err == ERANGE || value < min_value || value > max_value) {
                    if (flags & err_throw)
                        throw std::range_error("Integer out of range: " + quote(fragment));
                    t = sign_of(value) == 1 ? max_value : min_value;
                } else {
                    t = value;
                }
            } else {
{'fraction_non_alphanumeric': 0.12489494032610522, 'fraction_numerical': 0.01176668347621449, 'mean_word_length': 1.8799612778315586, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5261014', 'n_tokens_mistral': 1754, 'n_tokens_neox': 1631, 'n_words': 417}