<commit_before>/***********************************************************************
**
**  REBOL [R3] Language Interpreter and Run-time Environment
**
**  Copyright 2012 REBOL Technologies
**  REBOL is a trademark of REBOL Technologies
**
**  Licensed under the Apache License, Version 2.0 (the "License");
**  you may not use this file except in compliance with the License.
**  You may obtain a copy of the License at
**
**  http://www.apache.org/licenses/LICENSE-2.0
**
**  Unless required by applicable law or agreed to in writing, software
**  distributed under the License is distributed on an "AS IS" BASIS,
**  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
**  See the License for the specific language governing permissions and
**  limitations under the License.
**
************************************************************************
**
**  Module:  s-make.c
**  Summary: binary and unicode string support
**  Section: strings
**  Author:  <NAME>
**  Notes:
**
***********************************************************************/

#include "sys-core.h"
#include "sys-scan.h"


/***********************************************************************
**
*/	REBSER *Make_Binary(REBCNT length)
/*
**		Make a binary string series. For byte, C, and UTF8 strings.
**		Add 1 extra for terminator.
**
***********************************************************************/
{
	REBSER *series = Make_Series(length + 1, sizeof(REBYTE), FALSE);
	BIN_DATA(series)[length] = 0;
	return series;
}


/***********************************************************************
**
*/	REBSER *Make_Unicode(REBCNT length)
/*
**		Make a unicode string series. Used for internal strings.
**		Add 1 extra for terminator.
**
***********************************************************************/
{
	REBSER *series = Make_Series(length + 1, sizeof(REBUNI), FALSE);
	UNI_HEAD(series)[length] = 0;
	return series;
}


/***********************************************************************
**
*/	REBSER *Copy_Bytes(REBYTE *src, REBINT len)
/*
**		Create a string series from the given bytes.
**		Source is always latin-1 valid. Result is always 8bit.
**
***********************************************************************/
{
	REBSER *dst;

	if (len < 0) len = LEN_BYTES(src);

	dst = Make_Binary(len);
	memcpy(STR_DATA(dst), src, len);
	SERIES_TAIL(dst) = len;
	STR_TERM(dst);

	return dst;
}


/***********************************************************************
**
*/	REBSER *Copy_Bytes_To_Unicode(REBYTE *src, REBINT len)
/*
**		Convert a byte string to a unicode string. This can
**		be used for ASCII or LATIN-8 strings.
**
***********************************************************************/
{
	REBSER *series;
	REBUNI *dst;

	series = Make_Unicode(len);
	dst = UNI_HEAD(series);
	SERIES_TAIL(series) = len;

	for (; len > 0; len--) {
		*dst++ = (REBUNI)(*src++);
	}

	UNI_TERM(series);

	return series;
}


/***********************************************************************
**
*/	REBSER *Copy_OS_Str(void *src, REBINT len)
/*
**		Create a REBOL string series from an OS native string.
**
**		For example, in Win32 with the wide char interface, we must
**		convert wide char strings, minimizing to bytes if possible.
**
**		For Linux the char string could be UTF-8, so that must be
**		converted to REBOL Unicode or Latin byte strings.
**
***********************************************************************/
{
#ifdef OS_WIDE_CHAR
	REBSER *dst;
	if (Is_Wide((REBUNI*)src, len)) {
		REBUNI *up;
		dst = Make_Unicode(len);
		SERIES_TAIL(dst) = len;
		up = UNI_HEAD(dst);
		while (len-- > 0) *up++ = *((REBUNI*)src)++;
		*up = 0;
	}
	else {
		REBYTE *bp;
		dst = Make_Binary(len);
		SERIES_TAIL(dst) = len;
		bp = BIN_HEAD(dst);
		while (len-- > 0) *bp++ = (REBYTE)*((REBUNI*)src)++;
		*bp = 0;<commit_msg>fixed GCC compiler error: "lvalue required as increment operand"<commit_after>/***********************************************************************
**
**  REBOL [R3] Language Interpreter and Run-time Environment
**
**  Copyright 2012 REBOL Technologies
**  REBOL is a trademark of REBOL Technologies
**
**  Licensed under the Apache License, Version 2.0 (the "License");
**  you may not use this file except in compliance with the License.
**  You may obtain a copy of the License at
**
**  http://www.apache.org/licenses/LICENSE-2.0
**
**  Unless required by applicable law or agreed to in writing, software
**  distributed under the License is distributed on an "AS IS" BASIS,
**  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
**  See the License for the specific language governing permissions and
**  limitations under the License.
**
************************************************************************
**
**  Module:  s-make.c
**  Summary: binary and unicode string support
**  Section: strings
**  Author:  <NAME>
**  Notes:
**
***********************************************************************/

#include "sys-core.h"
#include "sys-scan.h"


/***********************************************************************
**
*/	REBSER *Make_Binary(REBCNT length)
/*
**		Make a binary string series. For byte, C, and UTF8 strings.
**		Add 1 extra for terminator.
**
***********************************************************************/
{
	REBSER *series = Make_Series(length + 1, sizeof(REBYTE), FALSE);
	BIN_DATA(series)[length] = 0;
	return series;
}


/***********************************************************************
**
*/	REBSER *Make_Unicode(REBCNT length)
/*
**		Make a unicode string series. Used for internal strings.
**		Add 1 extra for terminator.
**
***********************************************************************/
{
	REBSER *series = Make_Series(length + 1, sizeof(REBUNI), FALSE);
	UNI_HEAD(series)[length] = 0;
	return series;
}


/***********************************************************************
**
*/	REBSER *Copy_Bytes(REBYTE *src, REBINT len)
/*
**		Create a string series from the given bytes.
**		Source is always latin-1 valid. Result is always 8bit.
**
***********************************************************************/
{
	REBSER *dst;

	if (len < 0) len = LEN_BYTES(src);

	dst = Make_Binary(len);
	memcpy(STR_DATA(dst), src, len);
	SERIES_TAIL(dst) = len;
	STR_TERM(dst);

	return dst;
}


/***********************************************************************
**
*/	REBSER *Copy_Bytes_To_Unicode(REBYTE *src, REBINT len)
/*
**		Convert a byte string to a unicode string. This can
**		be used for ASCII or LATIN-8 strings.
**
***********************************************************************/
{
	REBSER *series;
	REBUNI *dst;

	series = Make_Unicode(len);
	dst = UNI_HEAD(series);
	SERIES_TAIL(series) = len;

	for (; len > 0; len--) {
		*dst++ = (REBUNI)(*src++);
	}

	UNI_TERM(series);

	return series;
}


/***********************************************************************
**
*/	REBSER *Copy_OS_Str(void *src, REBINT len)
/*
**		Create a REBOL string series from an OS native string.
**
**		For example, in Win32 with the wide char interface, we must
**		convert wide char strings, minimizing to bytes if possible.
**
**		For Linux the char string could be UTF-8, so that must be
**		converted to REBOL Unicode or Latin byte strings.
**
***********************************************************************/
{
#ifdef OS_WIDE_CHAR
	REBSER *dst;
	REBUNI *str = (REBUNI*)src;	
	if (Is_Wide(str, len)) {
		REBUNI *up;
		dst = Make_Unicode(len);
		SERIES_TAIL(dst) = len;
		up = UNI_HEAD(dst);
		while (len-- > 0) *up++ = *str++;
		*up = 0;
	}
	else {
		REBYTE *bp;
		dst = Make_Binary(len);
		SERIES_TAIL(dst) = len;
		bp = BIN_HEAD(dst);
		while (len-- > 0) *bp++ = (REBYTE)*str++;
		*bp = 0;
{'fraction_non_alphanumeric': 0.34682826199071687, 'fraction_numerical': 0.007220216606498195, 'mean_word_length': 5.1612390786338365, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6914167', 'n_tokens_mistral': 2455, 'n_tokens_neox': 2283, 'n_words': 696}