<commit_before>/*
 * Copyright (c) 2017 <NAME> <<EMAIL>>
 *
 * This file is part of fast-hessian, distributed under the MIT License
 * For full terms see the included LICENSE file
 */

#include "hessian.h"

static const size_t sublen = 0x8000;
static const uint16_t sublen_ns = 1; // htons(sublen)

short hessian_encode_string(v8::Local<v8::String> &str, uint8_t **out, size_t *len)
{
	size_t index = 0;
	int length = str->Length();
	int capacity = 3 * length + 10; // TODO 10?
	*out = (uint8_t*)malloc(capacity);
	if (NULL == *out) {
		return 0;
	}

	// TODO
	assert(length <= 0x8000);

	if (length <= 31) {
		(*out)[index++] = (uint8_t)(length);
	} else if (length <= 1023) {
		(*out)[index++] = (uint8_t)(48 + (length >> 8));
		(*out)[index++] = (uint8_t)(length); // Integer overflow and wrapping assumed
	} else {
		(*out)[index++] = 'S';
		(*out)[index++] = (uint8_t)((length >> 8));
		(*out)[index++] = (uint8_t)(length); // Integer overflow and wrapping assumed
	}

	int flags = v8::String::HINT_MANY_WRITES_EXPECTED |
		    v8::String::NO_NULL_TERMINATION |
		    v8::String::REPLACE_INVALID_UTF8;

	// TODO encoded as CESU-8
	int nbytes = str->WriteUtf8((char *)(*out), capacity, NULL, flags);
	*len = index + nbytes;
	*out = (uint8_t*)realloc(*out, *len);
	if (NULL == *out) {<commit_msg>fix typo<commit_after>/*
 * Copyright (c) 2017 <NAME> <<EMAIL>>
 *
 * This file is part of fast-hessian, distributed under the MIT License
 * For full terms see the included LICENSE file
 */

#include "hessian.h"

static const size_t sublen = 0x8000;
static const uint16_t sublen_ns = 1; // htons(sublen)

short hessian_encode_string(v8::Local<v8::String> &str, uint8_t **out, size_t *len)
{
	size_t index = 0;
	int length = str->Length();
	int capacity = 3 * length + 10; // TODO 10?
	*out = (uint8_t*)malloc(capacity);
	if (NULL == *out) {
		return 0;
	}

	// TODO
	assert(length <= 0x8000);

	if (length <= 31) {
		(*out)[index++] = (uint8_t)(length);
	} else if (length <= 1023) {
		(*out)[index++] = (uint8_t)(48 + (length >> 8));
		(*out)[index++] = (uint8_t)(length); // Integer overflow and wrapping assumed
	} else {
		(*out)[index++] = 'S';
		(*out)[index++] = (uint8_t)((length >> 8));
		(*out)[index++] = (uint8_t)(length); // Integer overflow and wrapping assumed
	}

	int flags = v8::String::HINT_MANY_WRITES_EXPECTED |
		    v8::String::NO_NULL_TERMINATION |
		    v8::String::REPLACE_INVALID_UTF8;

	// TODO encoded as CESU-8
	int nbytes = str->WriteUtf8((char *)(*out + index), capacity, NULL, flags);
	*len = index + nbytes;
	*out = (uint8_t*)realloc(*out, *len);
	if (NULL == *out) {
{'fraction_non_alphanumeric': 0.18800461361014995, 'fraction_numerical': 0.03844675124951941, 'mean_word_length': 4.162698412698413, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3789006', 'n_tokens_mistral': 1073, 'n_tokens_neox': 982, 'n_words': 254}