<commit_before>/*
 * Copyright 2017 Google Inc.
 *
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */

#include "include/private/SkSLString.h"
#include "include/private/SkStringView.h"

#include <cerrno>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <locale>
#include <memory>
#include <sstream>
#include <string>

std::string skstd::to_string(float value) {
    return skstd::to_string((double)value);
}

std::string skstd::to_string(double value) {
    std::stringstream buffer;
    buffer.imbue(std::locale::classic());
    buffer.precision(7);
    buffer << value;
    std::string text = buffer.str();

    float roundtripped;
    buffer >> roundtripped;
    if (roundtripped != (float)value) {
        buffer.str({});
        buffer.clear();
        buffer.precision(9);
        buffer << value;
        text = buffer.str();
        // TODO(johnstiles): on older versions of libc++, FLT_MAX will fill the buffer with the
        // correct value, but reading back into `roundtripped` gives INF. Disable the assertion
        // until we figure out a better way to check this.
        //SkASSERTF((buffer >> roundtripped, roundtripped == (float)value), "%.17g", value);
    }

    // We need to emit a decimal point to distinguish floats from ints.
    if (!skstd::contains(text, '.') && !skstd::contains(text, 'e')) {
        text += ".0";
    }

    return text;
}

bool SkSL::stod(std::string_view s, SKSL_FLOAT* value) {
    std::string str(s.data(), s.size());
    std::stringstream buffer(str);
    buffer.imbue(std::locale::classic());
    buffer >> *value;
    return !buffer.fail() && std::isfinite(*value);
}

bool SkSL::stoi(std::string_view s, SKSL_INT* value) {
    if (s.empty()) {
        return false;
    }
    char suffix = s.back();
    if (suffix == 'u' || suffix == 'U') {
        s.remove_suffix(1);
    }
    std::string str(s);  // s is not null-terminated
    const char* strEnd = str.data() + str.length();
    char* p;
    errno = 0;
    unsigned long long result = strtoull(str.data(), &p, /*base=*/0);
    *value = static_cast<SKSL_INT>(result);
    return p == strEnd && errno == 0 && result <= 0xFFFFFFFF;
}

std::string SkSL::String::printf(const char* fmt, ...) {
    va_list args;
    va_start(args, fmt);
    std::string result;
    vappendf(&result, fmt, args);
    va_end(args);
    return result;
}

void SkSL::String::appendf(std::string *str, const char* fmt, ...) {
    va_list args;
    va_start(args, fmt);
    vappendf(str, fmt, args);
    va_end(args);
}

void SkSL::String::vappendf(std::string *str, const char* fmt, va_list args) {
    #define BUFFER_SIZE 256
    char buffer[BUFFER_SIZE];
    va_list reuse;
    va_copy(reuse, args);
    size_t size = vsnprintf(buffer, BUFFER_SIZE, fmt, args);
    if (BUFFER_SIZE >= size + 1) {
        str->append(buffer, size);
    } else {
        auto newBuffer = std::unique_ptr<char[]>(new char[size + 1]);
        vsnprintf(newBuffer.get(), size + 1, fmt, reuse);
        str->append(newBuffer.get(), size);
    }
    va_end(reuse);
}
<commit_msg>Improve string-to-float roundtrip checking.<commit_after>/*
 * Copyright 2017 Google Inc.
 *
 * Use of this source code is governed by a BSD-style license that can be
 * found in the LICENSE file.
 */

#include "include/private/SkSLString.h"
#include "include/private/SkStringView.h"

#include <cerrno>
#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <locale>
#include <memory>
#include <sstream>
#include <string>

template <typename RoundtripType, int kFullPrecision>
static std::string to_string_impl(RoundtripType value) {
    std::stringstream buffer;
    buffer.imbue(std::locale::classic());
    buffer.precision(7);
    buffer << value;
    std::string text = buffer.str();

    double roundtripped;
    buffer >> roundtripped;
    if (value != (RoundtripType)roundtripped) {
        buffer.str({});
        buffer.clear();
        buffer.precision(kFullPrecision);
        buffer << value;
        text = buffer.str();
        SkASSERTF((buffer >> roundtripped, value == (RoundtripType)roundtripped),
                  "%.17g -> %s -> %.17g", value, text.c_str(), roundtripped);
    }

    // We need to emit a decimal point to distinguish floats from ints.
    if (!skstd::contains(text, '.') && !skstd::contains(text, 'e')) {
        text += ".0";
    }

    return text;
}

std::string skstd::to_string(float value) {
    return to_string_impl<float, 9>(value);
}

std::string skstd::to_string(double value) {
    return to_string_impl<double, 17>(value);
}

bool SkSL::stod(std::string_view s, SKSL_FLOAT* value) {
    std::string str(s.data(), s.size());
    std::stringstream buffer(str);
    buffer.imbue(std::locale::classic());
    buffer >> *value;
    return !buffer.fail() && std::isfinite(*value);
}

bool SkSL::stoi(std::string_view s, SKSL_INT* value) {
    if (s.empty()) {
        return false;
    }
    char suffix = s.back();
    if (suffix == 'u' || suffix == 'U') {
        s.remove_suffix(1);
    }
    std::string str(s);  // s is not null-terminated
    const char* strEnd = str.data() + str.length();
    char* p;
    errno = 0;
    unsigned long long result = strtoull(str.data(), &p, /*base=*/0);
    *value = static_cast<SKSL_INT>(result);
    return p == strEnd && errno == 0 && result <= 0xFFFFFFFF;
}

std::string SkSL::String::printf(const char* fmt, ...) {
    va_list args;
    va_start(args, fmt);
    std::string result;
    vappendf(&result, fmt, args);
    va_end(args);
    return result;
}

void SkSL::String::appendf(std::string *str, const char* fmt, ...) {
    va_list args;
    va_start(args, fmt);
    vappendf(str, fmt, args);
    va_end(args);
}

void SkSL::String::vappendf(std::string *str, const char* fmt, va_list args) {
    #define BUFFER_SIZE 256
    char buffer[BUFFER_SIZE];
    va_list reuse;
    va_copy(reuse, args);
    size_t size = vsnprintf(buffer, BUFFER_SIZE, fmt, args);
    if (BUFFER_SIZE >= size + 1) {
        str->append(buffer, size);
    } else {
        auto newBuffer = std::unique_ptr<char[]>(new char[size + 1]);
        vsnprintf(newBuffer.get(), size + 1, fmt, reuse);
        str->append(newBuffer.get(), size);
    }
    va_end(reuse);
}

{'fraction_non_alphanumeric': 0.16006389776357827, 'fraction_numerical': 0.007028753993610224, 'mean_word_length': 3.2218476062036414, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2064394', 'n_tokens_mistral': 2168, 'n_tokens_neox': 2092, 'n_words': 533}