<commit_before>/*
Copyright: 2001-2003 The Perl Foundation.  All Rights Reserved.
$Id$

=head1 NAME

encodings/utf16.c - UTF-16 encoding

=head1 DESCRIPTION

UTF-16 encoding with the help of the ICU library.

=head2 Functions

=over 4

=cut

*/

#include "parrot/parrot.h"
#include "parrot/unicode.h"

#include "utf16.h"

#if PARROT_HAS_ICU
#include <unicode/utf16.h>
#include <unicode/ustring.h>
#endif

#define UNIMPL internal_exception(UNIMPLEMENTED, "unimpl utf16")


static void iter_init(Interp *, String *src, String_iter *iter);
/* This function needs to go through and get all the code points one
   by one and turn them into a utf16 sequence */
static void
to_encoding(Interp *interpreter, STRING *src)
{
#if PARROT_HAS_ICU
    UErrorCode err;
    int dest_len;
    UChar *p;
#endif
    int src_len;

    if (src->encoding == Parrot_utf16_encoding_ptr)
        return;
    /*
     * TODO adapt string creation functions
     */
    src->charset  = Parrot_unicode_charset_ptr;
    src->encoding = Parrot_utf16_encoding_ptr;
    src_len = src->strlen;
    if (!src_len)
        return;
    /*
       u_strFromUTF8(UChar *dest,
       int32_t destCapacity,
       int32_t *pDestLength,
       const char *src,
       int32_t srcLength,
       UErrorCode *pErrorCode);
     */
#if PARROT_HAS_ICU
    err = U_ZERO_ERROR;
    /* XXX these inplace operations are all shit (sorry) */
    p = mem_sys_allocate(src_len * sizeof(UChar));
    u_strFromUTF8(p, src_len,
            &dest_len, src->strstart, src->bufused, &err);
    if (!U_SUCCESS(err)) {
        /*
         * have to resize - required len in UChars is in dest_len
         */
        p = mem_sys_realloc(p, dest_len * sizeof(UChar));
        u_strFromUTF8(p, dest_len,
                &dest_len, src->strstart, src->bufused, &err);
        assert(U_SUCCESS(err));
    }
    src->bufused = dest_len * sizeof(UChar);
    Parrot_reallocate_string(interpreter, src, src->bufused);
    memcpy(src->strstart, p, src->bufused);
    mem_sys_free(p);

    /* downgrade if possible */
    if (dest_len == (int)src->strlen)
        src->encoding = Parrot_ucs2_encoding_ptr;
#else
    real_exception(interpreter, NULL, E_LibraryNotLoadedError,
            "no ICU lib loaded");
#endif
}

static STRING *
copy_to_encoding(Interp *interpreter, STRING *src)
{
    STRING *dest;
#if PARROT_HAS_ICU
    UErrorCode err;
    int dest_len;
#endif
    int src_len;

    if (src->encoding == Parrot_utf16_encoding_ptr)
        return string_copy(interpreter, src);

    /*
     * TODO adapt string creation functions
     */
    dest = new_string_header(interpreter, 0);
    src_len = src->strlen;
    dest->strlen   = src_len;
    dest->charset  = Parrot_unicode_charset_ptr;
    dest->encoding = Parrot_utf16_encoding_ptr;
    if (!src_len) {
        Parrot_allocate_string(interpreter, dest, 0);
        return dest;
    }
#if PARROT_HAS_ICU
    Parrot_allocate_string(interpreter, dest, sizeof(UChar) * src_len);
    err = U_ZERO_ERROR;
    u_strFromUTF8(dest->strstart, src_len,
            &dest_len, src->strstart, src->bufused, &err);
    if (!U_SUCCESS(err)) {
        Parrot_allocate_string(interpreter, dest, sizeof(UChar) * dest_len);
        u_strFromUTF8(dest->strstart, dest_len,
                &dest_len, src->strstart, src->bufused, &err);
        assert(U_SUCCESS(err));
    }
    dest->bufused = dest_len * sizeof(UChar);
    /* downgrade if possible */
    if (dest_len == (int)src->strlen)
        src->encoding = Parrot_ucs2_encoding_ptr;

#else
    real_exception(interpreter, NULL, E_LibraryNotLoadedError,
            "no ICU lib loaded");
#endif

    return dest;
}

static UINTVAL
get_codepoint(Interp *interpreter, const STRING *src, UINTVAL offset)
{
    UChar *s = (UChar*) src->strstart;
    UINTVAL c, pos;

    pos = 0;
    U16_FWD_N_UNSAFE(s, pos, offset);
    U16_GET_UNSAFE(s, pos, c);
    return c;
}

static void
set_codepoint(Interp *interpreter, STRING *src,
	UINTVAL offset, UINTVAL codepoint)
{
    const void *start;
    void *p;
    UNIMPL;
}

static UINTVAL
get_byte(Interp *interpreter, const STRING *src, UINTVAL offset)
{
    unsigned char *contents = src->strstart;
    if (offset >= src->bufused) {
/*	internal_exception(0,
		"get_byte past the end of the buffer (%i of %i)",
		offset, src->bufused);*/
	return 0;
    }
    return contents[offset];
}

static void
set_byte(Interp *interpreter, const STRING *src,<commit_msg>fix unprotected usage of UChar in utf16.c<commit_after>/*
Copyright: 2001-2003 The Perl Foundation.  All Rights Reserved.
$Id$

=head1 NAME

encodings/utf16.c - UTF-16 encoding

=head1 DESCRIPTION

UTF-16 encoding with the help of the ICU library.

=head2 Functions

=over 4

=cut

*/

#include "parrot/parrot.h"
#include "parrot/unicode.h"

#include "utf16.h"

#if PARROT_HAS_ICU
#include <unicode/utf16.h>
#include <unicode/ustring.h>
#endif

#define UNIMPL internal_exception(UNIMPLEMENTED, "unimpl utf16")


static void iter_init(Interp *, String *src, String_iter *iter);
/* This function needs to go through and get all the code points one
   by one and turn them into a utf16 sequence */
static void
to_encoding(Interp *interpreter, STRING *src)
{
#if PARROT_HAS_ICU
    UErrorCode err;
    int dest_len;
    UChar *p;
#endif
    int src_len;

    if (src->encoding == Parrot_utf16_encoding_ptr)
        return;
    /*
     * TODO adapt string creation functions
     */
    src->charset  = Parrot_unicode_charset_ptr;
    src->encoding = Parrot_utf16_encoding_ptr;
    src_len = src->strlen;
    if (!src_len)
        return;
    /*
       u_strFromUTF8(UChar *dest,
       int32_t destCapacity,
       int32_t *pDestLength,
       const char *src,
       int32_t srcLength,
       UErrorCode *pErrorCode);
     */
#if PARROT_HAS_ICU
    err = U_ZERO_ERROR;
    /* XXX these inplace operations are all shit (sorry) */
    p = mem_sys_allocate(src_len * sizeof(UChar));
    u_strFromUTF8(p, src_len,
            &dest_len, src->strstart, src->bufused, &err);
    if (!U_SUCCESS(err)) {
        /*
         * have to resize - required len in UChars is in dest_len
         */
        p = mem_sys_realloc(p, dest_len * sizeof(UChar));
        u_strFromUTF8(p, dest_len,
                &dest_len, src->strstart, src->bufused, &err);
        assert(U_SUCCESS(err));
    }
    src->bufused = dest_len * sizeof(UChar);
    Parrot_reallocate_string(interpreter, src, src->bufused);
    memcpy(src->strstart, p, src->bufused);
    mem_sys_free(p);

    /* downgrade if possible */
    if (dest_len == (int)src->strlen)
        src->encoding = Parrot_ucs2_encoding_ptr;
#else
    real_exception(interpreter, NULL, E_LibraryNotLoadedError,
            "no ICU lib loaded");
#endif
}

static STRING *
copy_to_encoding(Interp *interpreter, STRING *src)
{
    STRING *dest;
#if PARROT_HAS_ICU
    UErrorCode err;
    int dest_len;
#endif
    int src_len;

    if (src->encoding == Parrot_utf16_encoding_ptr)
        return string_copy(interpreter, src);

    /*
     * TODO adapt string creation functions
     */
    dest = new_string_header(interpreter, 0);
    src_len = src->strlen;
    dest->strlen   = src_len;
    dest->charset  = Parrot_unicode_charset_ptr;
    dest->encoding = Parrot_utf16_encoding_ptr;
    if (!src_len) {
        Parrot_allocate_string(interpreter, dest, 0);
        return dest;
    }
#if PARROT_HAS_ICU
    Parrot_allocate_string(interpreter, dest, sizeof(UChar) * src_len);
    err = U_ZERO_ERROR;
    u_strFromUTF8(dest->strstart, src_len,
            &dest_len, src->strstart, src->bufused, &err);
    if (!U_SUCCESS(err)) {
        Parrot_allocate_string(interpreter, dest, sizeof(UChar) * dest_len);
        u_strFromUTF8(dest->strstart, dest_len,
                &dest_len, src->strstart, src->bufused, &err);
        assert(U_SUCCESS(err));
    }
    dest->bufused = dest_len * sizeof(UChar);
    /* downgrade if possible */
    if (dest_len == (int)src->strlen)
        src->encoding = Parrot_ucs2_encoding_ptr;

#else
    real_exception(interpreter, NULL, E_LibraryNotLoadedError,
            "no ICU lib loaded");
#endif

    return dest;
}

static UINTVAL
get_codepoint(Interp *interpreter, const STRING *src, UINTVAL offset)
{
#if PARROT_HAS_ICU
    UChar *s = (UChar*) src->strstart;
    UINTVAL c, pos;

    pos = 0;
    U16_FWD_N_UNSAFE(s, pos, offset);
    U16_GET_UNSAFE(s, pos, c);
    return c;
#else
    real_exception(interpreter, NULL, E_LibraryNotLoadedError,
            "no ICU lib loaded");
    return 0;
#endif
}

static void
set_codepoint(Interp *interpreter, STRING *src,
	UINTVAL offset, UINTVAL codepoint)
{
    const void *start;
    void *p;
    UNIMPL;
}

static UINTVAL
get_byte(Interp *interpreter, const STRING *src, UINTVAL offset)
{
    unsigned char *contents = src->strstart;
    if (offset >= src->bufused) {
/*	internal_exception(0,
		"get_byte past the end of the buffer (%i of %i)",
		offset, src->bufused);*/
	return 0;
    }
    return contents[offset];
}

static void
set_byte(Interp *interpreter, const STRING *src,
{'fraction_non_alphanumeric': 0.11244311244311245, 'fraction_numerical': 0.012765012765012766, 'mean_word_length': 3.1160347190497943, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 2, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7511479', 'n_tokens_mistral': 3269, 'n_tokens_neox': 3183, 'n_words': 766}