<commit_before>#include "testsettings.hpp"
#ifdef TEST_UTF8

#include <cstddef>
#include <limits>
#include <stdexcept>
#include <string>
#include <iostream>

#include <tightdb/util/assert.hpp>
#include <tightdb/util/unique_ptr.hpp>
#include <tightdb/util/utf8.hpp>
#include <tightdb/unicode.hpp>

#include "test.hpp"

using namespace std;
using namespace tightdb;
using namespace tightdb::util;
using namespace tightdb::test_util;

// Test independence and thread-safety
// -----------------------------------
//
// All tests must be thread safe and independent of each other. This
// is required because it allows for both shuffling of the execution
// order and for parallelized testing.
//
// In particular, avoid using std::rand() since it is not guaranteed
// to be thread safe. Instead use the API offered in
// `test/util/random.hpp`.
//
// All files created in tests must use the TEST_PATH macro (or one of
// its friends) to obtain a suitable file system path. See
// `test/util/test_path.hpp`.
//
//
// Debugging and the ONLY() macro
// ------------------------------
//
// A simple way of disabling all tests except one called `Foo`, is to
// replace TEST(Foo) with ONLY(Foo) and then recompile and rerun the
// test suite. Note that you can also use filtering by setting the
// environment varible `UNITTEST_FILTER`. See `README.md` for more on
// this.
//
// Another way to debug a particular test, is to copy that test into
// `experiments/testcase.cpp` and then run `sh build.sh
// check-testcase` (or one of its friends) from the command line.


namespace {

//const char* uY = "\x0CE\x0AB";              // greek capital letter upsilon with dialytika (U+03AB)
//const char* uYd = "\x0CE\x0A5\x0CC\x088";    // decomposed form (Y followed by two dots)
//const char* uy = "\x0CF\x08B";              // greek small letter upsilon with dialytika (U+03AB)
//const char* uyd = "\x0cf\x085\x0CC\x088";    // decomposed form (Y followed by two dots)

const char* uA = "\x0c3\x085";         // danish capital A with ring above (as in BLAABAERGROED)
//const char* uAd = "\x041\x0cc\x08a";    // decomposed form (A (41) followed by ring)
const char* ua = "\x0c3\x0a5";         // danish lower case a with ring above (as in blaabaergroed)
//const char* uad = "\x061\x0cc\x08a";    // decomposed form (a (41) followed by ring)

const char* uAE = "\xc3\x86"; // danish upper case AE
const char* uae = "\xc3\xa6"; // danish lower case ae

const char* u16sur = "\xF0\xA0\x9C\x8E"; // chineese needing utf16 surrogate pair
const char* u16sur2 = "\xF0\xA0\x9C\xB1"; // same as above, with larger unicode

TEST(UTF_Fuzzy_utf8_to_utf16)
{
    Random random(random_int<unsigned long>()); // Seed from slow global generator
    const size_t size = 10;
    char in[size];
    int16_t out[size];

    for (size_t iter = 0; iter < 100000; iter++) {
        for (size_t t = 0; t < size; t++) {
            in[t] = random.draw_int<char>();
        }

        const char* in2 = in;
        size_t needed = Utf8x16<int16_t>::find_utf16_buf_size(in2, in + size);
        size_t read = in2 - in;

        // number of utf16 codepoints should not exceed number of utf8 codepoints
        CHECK(needed <= size);
        
        // we should not read beyond input buffer
        CHECK(read <= size);

        int16_t* out2 = out;
        Utf8x16<int16_t>::to_utf16(in2, in2 + read, out2, out2 + needed);
        size_t read2 = in2 - in;
        size_t written = out2 - out;

        CHECK(read2 <= read);
        CHECK(written <= needed);
    }
}


TEST(UTF8_Compare_Core_ASCII) {
    // Useful line for creating new unit test cases:
    // bool ret = std::locale("us_EN")(string("a"), string("b"));

    set_string_compare_method(STRING_COMPARE_CORE, null_ptr);

    // simplest test
    CHECK_EQUAL(true, utf8_compare("a", "b"));
    CHECK_EQUAL(false, utf8_compare("b", "a"));
    CHECK_EQUAL(false, utf8_compare("a", "a"));

    // length makes a difference
    CHECK_EQUAL(true, utf8_compare("aaaa", "b"));
    CHECK_EQUAL(true, utf8_compare("a", "bbbb"));

    CHECK_EQUAL(true, utf8_compare("a", "aaaa"));<commit_msg>unit test fix<commit_after>#include "testsettings.hpp"
#ifdef TEST_UTF8

#include <cstddef>
#include <limits>
#include <stdexcept>
#include <string>
#include <iostream>

#include <tightdb/util/assert.hpp>
#include <tightdb/util/unique_ptr.hpp>
#include <tightdb/util/utf8.hpp>
#include <tightdb/unicode.hpp>

#include "test.hpp"

using namespace std;
using namespace tightdb;
using namespace tightdb::util;
using namespace tightdb::test_util;

// Test independence and thread-safety
// -----------------------------------
//
// All tests must be thread safe and independent of each other. This
// is required because it allows for both shuffling of the execution
// order and for parallelized testing.
//
// In particular, avoid using std::rand() since it is not guaranteed
// to be thread safe. Instead use the API offered in
// `test/util/random.hpp`.
//
// All files created in tests must use the TEST_PATH macro (or one of
// its friends) to obtain a suitable file system path. See
// `test/util/test_path.hpp`.
//
//
// Debugging and the ONLY() macro
// ------------------------------
//
// A simple way of disabling all tests except one called `Foo`, is to
// replace TEST(Foo) with ONLY(Foo) and then recompile and rerun the
// test suite. Note that you can also use filtering by setting the
// environment varible `UNITTEST_FILTER`. See `README.md` for more on
// this.
//
// Another way to debug a particular test, is to copy that test into
// `experiments/testcase.cpp` and then run `sh build.sh
// check-testcase` (or one of its friends) from the command line.


namespace {

//const char* uY = "\x0CE\x0AB";              // greek capital letter upsilon with dialytika (U+03AB)
//const char* uYd = "\x0CE\x0A5\x0CC\x088";    // decomposed form (Y followed by two dots)
//const char* uy = "\x0CF\x08B";              // greek small letter upsilon with dialytika (U+03AB)
//const char* uyd = "\x0cf\x085\x0CC\x088";    // decomposed form (Y followed by two dots)

const char* uA = "\x0c3\x085";         // danish capital A with ring above (as in BLAABAERGROED)
//const char* uAd = "\x041\x0cc\x08a";    // decomposed form (A (41) followed by ring)
const char* ua = "\x0c3\x0a5";         // danish lower case a with ring above (as in blaabaergroed)
//const char* uad = "\x061\x0cc\x08a";    // decomposed form (a (41) followed by ring)

const char* uAE = "\xc3\x86"; // danish upper case AE
const char* uae = "\xc3\xa6"; // danish lower case ae

const char* u16sur = "\xF0\xA0\x9C\x8E"; // chineese needing utf16 surrogate pair
const char* u16sur2 = "\xF0\xA0\x9C\xB1"; // same as above, with larger unicode

TEST(UTF_Fuzzy_utf8_to_utf16)
{
    Random random(random_int<unsigned long>()); // Seed from slow global generator
    const size_t size = 10;
    char in[size];
    int16_t out[size];

    for (size_t iter = 0; iter < 1000000; iter++) {
        for (size_t t = 0; t < size; t++) {
            in[t] = random.draw_int<char>();
        }

        const char* in2 = in;
        size_t needed = Utf8x16<int16_t>::find_utf16_buf_size(in2, in + size);
        size_t read = in2 - in;

        // number of utf16 codepoints should not exceed number of utf8 codepoints
        CHECK(needed <= size);
        
        // we should not read beyond input buffer
        CHECK(read <= size);

        int16_t* out2 = out;
        in2 = in;
        Utf8x16<int16_t>::to_utf16(in2, in2 + read, out2, out2 + needed);
        size_t read2 = in2 - in;
        size_t written = out2 - out;

        CHECK(read2 <= size);
        CHECK(written <= needed);
    }
}


TEST(UTF8_Compare_Core_ASCII) {
    // Useful line for creating new unit test cases:
    // bool ret = std::locale("us_EN")(string("a"), string("b"));

    set_string_compare_method(STRING_COMPARE_CORE, null_ptr);

    // simplest test
    CHECK_EQUAL(true, utf8_compare("a", "b"));
    CHECK_EQUAL(false, utf8_compare("b", "a"));
    CHECK_EQUAL(false, utf8_compare("a", "a"));

    // length makes a difference
    CHECK_EQUAL(true, utf8_compare("aaaa", "b"));
    CHECK_EQUAL(true, utf8_compare("a", "bbbb"));

    CHECK_EQUAL(true, utf8_compare("a", "aaaa"));
{'fraction_non_alphanumeric': 0.14547662416514875, 'fraction_numerical': 0.030358227079538554, 'mean_word_length': 2.8271375464684017, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5678912', 'n_tokens_mistral': 2871, 'n_tokens_neox': 2707, 'n_words': 899}