<commit_before>/* $chaos: unicode.c,v 1.1 2002/06/23 20:34:44 per Exp $ */
/* Abstract: Unicode library. */
/* Authors: <NAME> <<EMAIL>>
            <NAME> <<EMAIL>> */

/* Copyright 2000-2002 chaos development */
/* Use freely under the terms listed in the file LICENSE. */

/* See the specification (RFC 2279) for more information about
   UTF-8. If you are looking for general Unicode information, try out
   http://www.unicode.org/ */

#include <unicode/unicode.h>

#define B00000000 0x00
#define B10000000 0x80
#define B11000000 0xC0
#define B11100000 0xE0
#define B11110000 0xF0
#define B11111000 0xF8
#define B11111100 0xFC
#define B11111110 0xFE

#define B00111111 0x3F
#define B00011111 0x1F
#define B00001111 0x0F
#define B00000111 0x07
#define B00000011 0x03
#define B00000001 0x01

/* Convert a UCS-2 character to UTF-8. */
return_t unicode_ucs2_to_utf8 (uint8_t *utf8_char, ucs2_t ucs2_char)
{
    int utf8_bytes;

    if (ucs2_char <= 0x0000007F)
    {
        /* 0xxxxxxx */
        utf8_bytes = 1;
        utf8_char[0] = (uint8_t) ucs2_char;
    }
    else if (ucs2_char <= 0x000007FF)
    {
        /* 110xxxxx 10xxxxxx */
        utf8_bytes = 2;
        utf8_char[0] = (uint8_t) ((ucs2_char >> 6) | B11000000);
        utf8_char[1] = (uint8_t) ((ucs2_char & B00111111) | B10000000);
    }
    else if (ucs2_char <= 0x0000FFFF)
    {
        /* 1110xxxx 10xxxxxx 10xxxxxx */
        utf8_bytes = 3;
        utf8_char[0] = (uint8_t) ((ucs2_char >> 12) | B11100000);
        utf8_char[1] = (uint8_t) (((ucs2_char >> 6) & B00111111) | B10000000);
        utf8_char[2] = (uint8_t) ((ucs2_char & B00111111) | B10000000);
    }
    else
    {
        return UNICODE_RETURN_UCS2_INVALID;
    }

    utf8_char[utf8_bytes] = '\0';

    return UNICODE_RETURN_SUCCESS;
}

/* Convert a UCS-4 character to UTF-8. */
return_t unicode_ucs4_to_utf8 (uint8_t *utf8_char, ucs4_t ucs4_char)
{
    int utf8_bytes;

    if (ucs4_char <= 0x0000007F)<commit_msg>Commented out code because of gcc warnings<commit_after>/* $chaos: unicode.c,v 1.1 2002/06/23 20:34:44 per Exp $ */
/* Abstract: Unicode library. */
/* Authors: <NAME> <<EMAIL>>
            <NAME> <<EMAIL>> */

/* Copyright 2000-2002 chaos development */
/* Use freely under the terms listed in the file LICENSE. */

/* See the specification (RFC 2279) for more information about
   UTF-8. If you are looking for general Unicode information, try out
   http://www.unicode.org/ */

#include <unicode/unicode.h>

#define B00000000 0x00
#define B10000000 0x80
#define B11000000 0xC0
#define B11100000 0xE0
#define B11110000 0xF0
#define B11111000 0xF8
#define B11111100 0xFC
#define B11111110 0xFE

#define B00111111 0x3F
#define B00011111 0x1F
#define B00001111 0x0F
#define B00000111 0x07
#define B00000011 0x03
#define B00000001 0x01

/* Convert a UCS-2 character to UTF-8. */
return_t unicode_ucs2_to_utf8 (uint8_t *utf8_char, ucs2_t ucs2_char)
{
    int utf8_bytes;

    if (ucs2_char <= 0x0000007F)
    {
        /* 0xxxxxxx */
        utf8_bytes = 1;
        utf8_char[0] = (uint8_t) ucs2_char;
    }
    else if (ucs2_char <= 0x000007FF)
    {
        /* 110xxxxx 10xxxxxx */
        utf8_bytes = 2;
        utf8_char[0] = (uint8_t) ((ucs2_char >> 6) | B11000000);
        utf8_char[1] = (uint8_t) ((ucs2_char & B00111111) | B10000000);
    }
    else /* if (ucs2_char <= 0x0000FFFF) */ /* had to remove this because gcc barfed about it, always true because of int16 */
    {
        /* 1110xxxx 10xxxxxx 10xxxxxx */
        utf8_bytes = 3;
        utf8_char[0] = (uint8_t) ((ucs2_char >> 12) | B11100000);
        utf8_char[1] = (uint8_t) (((ucs2_char >> 6) & B00111111) | B10000000);
        utf8_char[2] = (uint8_t) ((ucs2_char & B00111111) | B10000000);
    }
    /* else
    {
        return UNICODE_RETURN_UCS2_INVALID;
    } */

    utf8_char[utf8_bytes] = '\0';

    return UNICODE_RETURN_SUCCESS;
}

/* Convert a UCS-4 character to UTF-8. */
return_t unicode_ucs4_to_utf8 (uint8_t *utf8_char, ucs4_t ucs4_char)
{
    int utf8_bytes;

    if (ucs4_char <= 0x0000007F)
{'fraction_non_alphanumeric': 0.11917740336967295, 'fraction_numerical': 0.16798810703666997, 'mean_word_length': 3.0009910802775024, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 16, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1667494', 'n_tokens_mistral': 2046, 'n_tokens_neox': 1579, 'n_words': 389}