<commit_before>/*
 * Copyright (c) 2021, <NAME> <<EMAIL>>
 *
 * SPDX-License-Identifier: GPL-3.0-or-later
 */

//
// Created by <NAME> on 2021-09-20.
//

#include <cassert>
#include <cctype>

#include <core/StringUtil.h>

namespace Obelix {

int stricmp(const char* a, const char* b)
{
    unsigned char ca, cb;
    do {
        ca = tolower(toupper((unsigned char)*a++));
        cb = tolower(toupper((unsigned char)*b++));
    } while (ca == cb && ca != '\0');
    return ca - cb;
}

std::string to_upper(std::string const& input)
{
    std::string ret;
    for (auto& ch : input) {
        ret += (char) toupper((int) ch);
    }
    return ret;
}

std::string to_lower(std::string const& input)
{
    std::string ret;
    for (auto& ch : input) {
        ret += (char) tolower((int) ch);
    }
    return ret;
}

std::string c_escape(std::string const& s)
{
    std::string ret;
    for (char c : s) {
        if (c == '"' || c == '\'' || c == '\\') {
            ret += "\\";
        }
        ret += c;
    }
    return ret;
}

std::vector<std::string> split(std::string const& s, char sep)
{
    auto start = 0u;
    auto ptr = 0u;
    std::vector<std::string> ret;
    do {
        start = ptr;
        for (; (ptr < s.length()) && (s[ptr] != sep); ptr++)
            ;
        ret.push_back(s.substr(start, ptr - start));
        start = ++ptr;
    } while (ptr < s.length());
    if (s[s.length() - 1] == sep) // This is ugly...
        ret.emplace_back("");
    return ret;
}

std::string join(std::vector<std::string> const& collection, std::string const& sep)
{
    std::string ret;
    auto first = true;
    for (auto& elem : collection) {
        if (!first) {
            ret += sep;
        }
        ret += elem;
        first = false;
    }
    return ret;
}

std::string join(std::vector<std::string> const& collection, char sep)
{
    std::string sep_str;
    sep_str += sep;
    return join(collection, sep_str);
}

std::string strip(std::string const& s)
{
    size_t start;
    for (start = 0; (start < s.length()) && std::isspace(s[start]); start++)
        ;
    if (start == s.length()) {
        return {};
    }
    size_t end;
    for (end = s.length() - 1; std::isspace(s[end]); end--)
        ;
    return s.substr(start, end - start + 1);
}

std::string rstrip(std::string const& s)
{
    size_t end;
    for (end = s.length() - 1; std::isspace(s[end]); end--)
        ;
    return s.substr(0, end + 1);
}

std::string lstrip(std::string const& s)
{
    size_t start;
    for (start = 0; (start < s.length()) && std::isspace(s[start]); start++)
        ;
    return s.substr(start, s.length() - start);
}

std::vector<std::pair<std::string, std::string>> parse_pairs(std::string const& s, char pair_sep, char name_value_sep)
{
    auto pairs = split(s, pair_sep);
    std::vector<std::pair<std::string, std::string>> ret;
    for (auto& pair : pairs) {
        auto nvp = split(strip(pair), name_value_sep);
        switch (nvp.size()) {
        case 0:
            break;
        case 1:
            if (!strip(nvp[0]).empty())
                ret.emplace_back(strip(nvp[0]), "");
            break;
        case 2:
            if (!strip(nvp[0]).empty())
                ret.emplace_back(strip(nvp[0]), strip(nvp[1]));
            break;
        default:
            if (!strip(nvp[0]).empty()) {
                std::vector<std::string> tail;
                for (auto ix = 1u; ix < nvp.size(); ix++) {
                    tail.push_back(nvp[ix]);
                }
                auto value = strip(join(tail, name_value_sep));
                ret.emplace_back(strip(nvp[0]), strip(value));
            }
        }
    }
    return ret;
}

std::string to_string(long value)
{
    char buf[80];
    snprintf(buf, 79, "%ld", value);
    return buf;
}

std::string to_hex_string(long value)
{
    char buf[80];
    snprintf(buf, 79, "%lx", value);
    return buf;
}

std::string to_string(double value)
{
    char buf[80];
    snprintf(buf, 79, "%f", value);
    return buf;
}

std::string to_string(bool value)
{
    return (value) ? "true" : "false";
}

bool check_zero(std::string const& str)
{
    int zeroes = 0;
    for (auto& ch : str) {
        switch (ch) {
        case 'x':
        case 'X':
            if (zeroes != 1)
                return false;
            break;
        case '0':
            zeroes += (ch == '0') ? 1 : 0;
            break;
        default:
            return false;<commit_msg>$0000 wouldn't parse as a zero hex value<commit_after>/*
 * Copyright (c) 2021, <NAME> <<EMAIL>>
 *
 * SPDX-License-Identifier: GPL-3.0-or-later
 */

//
// Created by <NAME> on 2021-09-20.
//

#include <cassert>
#include <cctype>

#include <core/StringUtil.h>

namespace Obelix {

int stricmp(const char* a, const char* b)
{
    unsigned char ca, cb;
    do {
        ca = tolower(toupper((unsigned char)*a++));
        cb = tolower(toupper((unsigned char)*b++));
    } while (ca == cb && ca != '\0');
    return ca - cb;
}

std::string to_upper(std::string const& input)
{
    std::string ret;
    for (auto& ch : input) {
        ret += (char) toupper((int) ch);
    }
    return ret;
}

std::string to_lower(std::string const& input)
{
    std::string ret;
    for (auto& ch : input) {
        ret += (char) tolower((int) ch);
    }
    return ret;
}

std::string c_escape(std::string const& s)
{
    std::string ret;
    for (char c : s) {
        if (c == '"' || c == '\'' || c == '\\') {
            ret += "\\";
        }
        ret += c;
    }
    return ret;
}

std::vector<std::string> split(std::string const& s, char sep)
{
    auto start = 0u;
    auto ptr = 0u;
    std::vector<std::string> ret;
    do {
        start = ptr;
        for (; (ptr < s.length()) && (s[ptr] != sep); ptr++)
            ;
        ret.push_back(s.substr(start, ptr - start));
        start = ++ptr;
    } while (ptr < s.length());
    if (s[s.length() - 1] == sep) // This is ugly...
        ret.emplace_back("");
    return ret;
}

std::string join(std::vector<std::string> const& collection, std::string const& sep)
{
    std::string ret;
    auto first = true;
    for (auto& elem : collection) {
        if (!first) {
            ret += sep;
        }
        ret += elem;
        first = false;
    }
    return ret;
}

std::string join(std::vector<std::string> const& collection, char sep)
{
    std::string sep_str;
    sep_str += sep;
    return join(collection, sep_str);
}

std::string strip(std::string const& s)
{
    size_t start;
    for (start = 0; (start < s.length()) && std::isspace(s[start]); start++)
        ;
    if (start == s.length()) {
        return {};
    }
    size_t end;
    for (end = s.length() - 1; std::isspace(s[end]); end--)
        ;
    return s.substr(start, end - start + 1);
}

std::string rstrip(std::string const& s)
{
    size_t end;
    for (end = s.length() - 1; std::isspace(s[end]); end--)
        ;
    return s.substr(0, end + 1);
}

std::string lstrip(std::string const& s)
{
    size_t start;
    for (start = 0; (start < s.length()) && std::isspace(s[start]); start++)
        ;
    return s.substr(start, s.length() - start);
}

std::vector<std::pair<std::string, std::string>> parse_pairs(std::string const& s, char pair_sep, char name_value_sep)
{
    auto pairs = split(s, pair_sep);
    std::vector<std::pair<std::string, std::string>> ret;
    for (auto& pair : pairs) {
        auto nvp = split(strip(pair), name_value_sep);
        switch (nvp.size()) {
        case 0:
            break;
        case 1:
            if (!strip(nvp[0]).empty())
                ret.emplace_back(strip(nvp[0]), "");
            break;
        case 2:
            if (!strip(nvp[0]).empty())
                ret.emplace_back(strip(nvp[0]), strip(nvp[1]));
            break;
        default:
            if (!strip(nvp[0]).empty()) {
                std::vector<std::string> tail;
                for (auto ix = 1u; ix < nvp.size(); ix++) {
                    tail.push_back(nvp[ix]);
                }
                auto value = strip(join(tail, name_value_sep));
                ret.emplace_back(strip(nvp[0]), strip(value));
            }
        }
    }
    return ret;
}

std::string to_string(long value)
{
    char buf[80];
    snprintf(buf, 79, "%ld", value);
    return buf;
}

std::string to_hex_string(long value)
{
    char buf[80];
    snprintf(buf, 79, "%lx", value);
    return buf;
}

std::string to_string(double value)
{
    char buf[80];
    snprintf(buf, 79, "%f", value);
    return buf;
}

std::string to_string(bool value)
{
    return (value) ? "true" : "false";
}

bool check_zero(std::string const& str)
{
    int zeroes = 0;
    for (auto& ch : str) {
        switch (ch) {
        case '$':
            if (zeroes != 0)
                return false;
            break;
        case 'x':
        case 'X':
            if (zeroes != 1)
                return false;
            break;
        case '0':
            zeroes += (ch == '0') ? 1 : 0;
            break;
        default:
            return false;
{'fraction_non_alphanumeric': 0.17732461487310208, 'fraction_numerical': 0.012523550925412835, 'mean_word_length': 2.049678945589726, 'pattern_counts': {'":': 0, '<': 45, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5489475', 'n_tokens_mistral': 3239, 'n_tokens_neox': 3128, 'n_words': 769}