<commit_before>#include "iconv.h"

#include <v8.h>
#include <node.h>
#include <node_buffer.h>

#include <strings.h>	// strcasecmp + strncasecmp
#include <string.h>
#include <errno.h>

using namespace v8;
using namespace node;

namespace {

class Iconv: public ObjectWrap {
public:
	static void Initialize(Handle<Object>& target);
	static Handle<Value> New(const Arguments& args);
	static Handle<Value> Convert(const Arguments& args);

	Iconv(iconv_t conv);
	~Iconv(); // destructor may not run if program is short-lived or aborted

	// the actual conversion happens here
	Handle<Value> Convert(char* data, size_t length);

private:
	iconv_t conv_;
};

Iconv::Iconv(iconv_t conv): conv_(conv) {
	assert(conv_ != (iconv_t) -1);
}

Iconv::~Iconv() {
	iconv_close(conv_);
}

// helper class: reverse linked list of dumb buffers
struct chunk {
	chunk *const prev;
	size_t size;
	char data[32 * 1024];

	chunk(chunk *prev): prev(prev), size(0) {
	}
};

// the actual conversion happens here
Handle<Value> Iconv::Convert(char* data, size_t length) {
	assert(conv_ != (iconv_t) -1);
	assert(data != 0);

	chunk *c = 0;
	char *inbuf = data;
	size_t inbytesleft = length;
	size_t offset = 0;

	while (true) {
		c = new chunk(c);

		char *outbuf = c->data;
		size_t outbytesleft = sizeof(c->data);

		size_t rv = iconv(conv_, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
		c->size = sizeof(c->data) - outbytesleft;
		offset += c->size;

		if (rv != (size_t) -1) {
			assert(inbytesleft == 0);
			break;
		}

		if (errno == E2BIG) {
			continue;
		}

		if (errno == EINVAL) {
			// write out shift sequences (if any)
			rv = iconv(conv_, 0, 0, &outbuf, &outbytesleft);
			if (errno == E2BIG) {
				// chunk is too small to contain the shift sequence, retry with new chunk
				c = new chunk(c);
				outbuf = c->data;
				outbytesleft = sizeof(c->data);
				rv = iconv(conv_, 0, 0, &outbuf, &outbytesleft);
			}
			// we're still in an error condition if iconv() hasn't written any bytes
			if (rv != 0 && rv != (size_t) -1) {
				break;
			}
			if (rv == 0 || errno == EINVAL) {
				return ThrowException(ErrnoException(EINVAL, "iconv", "Incomplete character sequence."));
			}
			// deliberate fall through
		}

		if (errno == EILSEQ) {
			return ThrowException(ErrnoException(errno, "iconv", "Illegal character sequence."));
		}

		return ThrowException(ErrnoException(errno, "iconv"));
	}

	// copy linked list of chunks into Buffer in reverse order (last chunk at the top, second-to-last chunk below that, etc)
	Buffer& b = *Buffer::New(offset);
	for (chunk *t; c != 0; t = c->prev, delete c, c = t) {
		offset -= c->size;
		memcpy(b.data() + offset, c->data, c->size);
	}

	return b.handle_;
}

Handle<Value> Iconv::Convert(const Arguments& args) {
	HandleScope scope;

	Iconv* self = ObjectWrap::Unwrap<Iconv>(args.This());<commit_msg>Fix a nasty memory leak under error conditions.<commit_after>#include "iconv.h"

#include <v8.h>
#include <node.h>
#include <node_buffer.h>

#include <strings.h>	// strcasecmp + strncasecmp
#include <string.h>
#include <errno.h>

#include <list>

using namespace v8;
using namespace node;

namespace {

class Iconv: public ObjectWrap {
public:
	static void Initialize(Handle<Object>& target);
	static Handle<Value> New(const Arguments& args);
	static Handle<Value> Convert(const Arguments& args);

	Iconv(iconv_t conv);
	~Iconv(); // destructor may not run if program is short-lived or aborted

	// the actual conversion happens here
	Handle<Value> Convert(char* data, size_t length);

private:
	iconv_t conv_;
};

Iconv::Iconv(iconv_t conv): conv_(conv) {
	assert(conv_ != (iconv_t) -1);
}

Iconv::~Iconv() {
	iconv_close(conv_);
}

// helper class: reverse linked list of dumb buffers
struct chunk {
	size_t size;
	char data[32 * 1024];

	chunk(): size(0) {
	}
};

// the actual conversion happens here
Handle<Value> Iconv::Convert(char* data, size_t length) {
	assert(conv_ != (iconv_t) -1);
	assert(data != 0);

	std::list<chunk> chunks;

	char *inbuf = data;
	size_t inbytesleft = length;
	size_t size = 0;

	int index = 0;
	while (true) {
		// smart compilers will optimize chunks.push_back(chunk()) down to a zero-copy operation
		// but we'll assume the worst and do it manually
		chunks.resize(++index);
		chunk& c = chunks.back();

		char *outbuf = c.data;
		size_t outbytesleft = sizeof(c.data);

		size_t rv = iconv(conv_, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
		c.size = sizeof(c.data) - outbytesleft;
		size += c.size;

		if (rv != (size_t) -1) {
			assert(inbytesleft == 0);
			break;
		}

		if (errno == E2BIG) {
			continue;
		}

		if (errno == EINVAL) {
			// write out shift sequences (if any)
			rv = iconv(conv_, 0, 0, &outbuf, &outbytesleft);
			if (errno == E2BIG) {
				// chunk is too small to contain the shift sequence, retry with new chunk
				chunks.resize(++index);
				chunk& c = chunks.back();	// FIXME shadowing another variable is bad style

				outbuf = c.data;
				outbytesleft = sizeof(c.data);

				rv = iconv(conv_, 0, 0, &outbuf, &outbytesleft);
				if (rv != (size) -1) {
					c.size = rv;
				}
			}

			// we're still in an error condition if iconv() hasn't written any bytes
			if (rv != 0 && rv != (size_t) -1) {
				break;
			}

			if (rv == 0 || errno == EINVAL) {
				return ThrowException(ErrnoException(EINVAL, "iconv", "Incomplete character sequence."));
			}
			// deliberate fall through
		}

		if (errno == EILSEQ) {
			return ThrowException(ErrnoException(errno, "iconv", "Illegal character sequence."));
		}

		return ThrowException(ErrnoException(errno, "iconv"));
	}

	// copy chunks into buffer
	Buffer& b = *Buffer::New(size);

	char* p = b.data();
	for (std::list<chunk>::const_iterator chunk = chunks.begin(), end = chunks.end(); chunk != end; ++chunk) {
		memcpy(p, chunk->data, chunk->size);
		p += chunk->size;
	}

	return b.handle_;
}

Handle<Value> Iconv::Convert(const Arguments& args) {
	HandleScope scope;

	Iconv* self = ObjectWrap::Unwrap<Iconv>(args.This());
{'fraction_non_alphanumeric': 0.1507537688442211, 'fraction_numerical': 0.008710217755443886, 'mean_word_length': 3.738888888888889, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5656469', 'n_tokens_mistral': 2330, 'n_tokens_neox': 2097, 'n_words': 621}