<commit_before>#include "tl_string.h"

#include <string.h>



#define LEAD_OFFSET (0xD800 - (0x10000 >> 10))
#define SURROGATE_OFFSET (0x10000 - (0xD800 << 10) - 0xDC00)

#define IS_LEAD_SURROGATE( x ) (((x)>=0xD800) && ((x)<=0xDBFF))
#define IS_TRAIL_SURROGATE( x ) (((x)>=0xDC00) && ((x)<=0xDFFF))

#define IS_SURROGATE( x ) (((x) >= 0xD800) && ((x) <= 0xDFFF))
#define UNICODE_MAX 0x0010FFFF
#define REPLACEMENT_CHAR 0xFFFD
#define BOM 0xFEFF
#define BOM2 0xFFFE



/*
    determine how many UTF-16 code units are required to
    encode a given UTF-8 string
 */
static size_t utf8_count( const unsigned char* str, size_t chars )
{
    size_t i=0, count = 0;

    while( *str && i<chars )
    {
        if( (*str & 0xC0) != 0x80 )
        {
            count += ((*str & 0xF8)==0xF0) ? 2 : 1;
            ++i;
        }

        ++str;
    }

    return count;
}

static size_t utf8_strlen( const char* utf8 )
{
    const unsigned char* str = (const unsigned char*)utf8;
    size_t count = 0;

    while( *str )
    {
        if( (*str & 0xC0) != 0x80 )
            ++count;
        ++str;
    }

    return count;
}

static size_t utf16_strlen( const uint16_t* str )
{
    size_t count = 0;

    while( *str )
    {
        if( IS_LEAD_SURROGATE( *str ) )
        {
            if( IS_TRAIL_SURROGATE( str[1] ) )
            {
                str += 2;
                ++count;
                continue;
            }
        }
        ++count;
        ++str;
    }

    return count;
}

static size_t utf16_code_count( const uint16_t* str, size_t chars )
{
    size_t i, count = 0;

    for( i=0; i<chars; ++i )
    {
        if( IS_LEAD_SURROGATE( *str ) )
        {
            if( IS_TRAIL_SURROGATE( str[1] ) )
            {
                count += 2;
                str += 2;
                continue;
            }
        }
        count += 1;
        str += 1;
    }

    return count;
}

/****************************************************************************/

int tl_string_init( tl_string* this )
{
    uint16_t null = 0;

    if( this )
    {
        tl_array_init( &(this->vec), sizeof(uint16_t) );
        this->charcount = 0;
        this->surrogates = 0;

        if( tl_array_append( &(this->vec), &null ) )
            return 1;
    }

    return 0;
}

void tl_string_cleanup( tl_string* this )
{
    if( this )
    {
        tl_array_cleanup( &(this->vec) );

        this->charcount = 0;
        this->surrogates = 0;
    }
}

int tl_string_copy( tl_string* this, const tl_string* src )
{
    if( tl_array_copy( &this->vec, &src->vec ) )
        return 0;

    this->charcount  = src->charcount;
    this->surrogates = src->surrogates;
    return 1;
}

size_t tl_string_characters( const tl_string* this )
{
    return this ? this->charcount : 0;
}

size_t tl_string_length( const tl_string* this )
{
    return this ? (this->vec.used - 1) : 0;
}

void tl_string_clear( tl_string* this )
{
    if( this )
    {
        tl_array_resize( &(this->vec), 1 );
        *((uint16_t*)this->vec.data) = 0;

        this->charcount = 0;
        this->surrogates = 0;
    }
}

int tl_string_is_empty( const tl_string* this )
{<commit_msg>[fix] string copy<commit_after>#include "tl_string.h"

#include <string.h>



#define LEAD_OFFSET (0xD800 - (0x10000 >> 10))
#define SURROGATE_OFFSET (0x10000 - (0xD800 << 10) - 0xDC00)

#define IS_LEAD_SURROGATE( x ) (((x)>=0xD800) && ((x)<=0xDBFF))
#define IS_TRAIL_SURROGATE( x ) (((x)>=0xDC00) && ((x)<=0xDFFF))

#define IS_SURROGATE( x ) (((x) >= 0xD800) && ((x) <= 0xDFFF))
#define UNICODE_MAX 0x0010FFFF
#define REPLACEMENT_CHAR 0xFFFD
#define BOM 0xFEFF
#define BOM2 0xFFFE



/*
    determine how many UTF-16 code units are required to
    encode a given UTF-8 string
 */
static size_t utf8_count( const unsigned char* str, size_t chars )
{
    size_t i=0, count = 0;

    while( *str && i<chars )
    {
        if( (*str & 0xC0) != 0x80 )
        {
            count += ((*str & 0xF8)==0xF0) ? 2 : 1;
            ++i;
        }

        ++str;
    }

    return count;
}

static size_t utf8_strlen( const char* utf8 )
{
    const unsigned char* str = (const unsigned char*)utf8;
    size_t count = 0;

    while( *str )
    {
        if( (*str & 0xC0) != 0x80 )
            ++count;
        ++str;
    }

    return count;
}

static size_t utf16_strlen( const uint16_t* str )
{
    size_t count = 0;

    while( *str )
    {
        if( IS_LEAD_SURROGATE( *str ) )
        {
            if( IS_TRAIL_SURROGATE( str[1] ) )
            {
                str += 2;
                ++count;
                continue;
            }
        }
        ++count;
        ++str;
    }

    return count;
}

static size_t utf16_code_count( const uint16_t* str, size_t chars )
{
    size_t i, count = 0;

    for( i=0; i<chars; ++i )
    {
        if( IS_LEAD_SURROGATE( *str ) )
        {
            if( IS_TRAIL_SURROGATE( str[1] ) )
            {
                count += 2;
                str += 2;
                continue;
            }
        }
        count += 1;
        str += 1;
    }

    return count;
}

/****************************************************************************/

int tl_string_init( tl_string* this )
{
    uint16_t null = 0;

    if( this )
    {
        tl_array_init( &(this->vec), sizeof(uint16_t) );
        this->charcount = 0;
        this->surrogates = 0;

        if( tl_array_append( &(this->vec), &null ) )
            return 1;
    }

    return 0;
}

void tl_string_cleanup( tl_string* this )
{
    if( this )
    {
        tl_array_cleanup( &(this->vec) );

        this->charcount = 0;
        this->surrogates = 0;
    }
}

int tl_string_copy( tl_string* this, const tl_string* src )
{
    if( !tl_array_copy( &this->vec, &src->vec ) )
        return 0;

    this->charcount  = src->charcount;
    this->surrogates = src->surrogates;
    return 1;
}

size_t tl_string_characters( const tl_string* this )
{
    return this ? this->charcount : 0;
}

size_t tl_string_length( const tl_string* this )
{
    return this ? (this->vec.used - 1) : 0;
}

void tl_string_clear( tl_string* this )
{
    if( this )
    {
        tl_array_resize( &(this->vec), 1 );
        *((uint16_t*)this->vec.data) = 0;

        this->charcount = 0;
        this->surrogates = 0;
    }
}

int tl_string_is_empty( const tl_string* this )
{
{'fraction_non_alphanumeric': 0.16110671936758894, 'fraction_numerical': 0.0366798418972332, 'mean_word_length': 1.968559361801971, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1818069', 'n_tokens_mistral': 2585, 'n_tokens_neox': 2381, 'n_words': 532}