<commit_before>package se.culvertsoft.mgen.javapack.util;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.CoderResult;
import java.nio.charset.CodingErrorAction;

public class StringEncoder {

	private final ByteBuffer m_buffer;

	private final CharsetEncoder m_stringEncoder;

	public StringEncoder(final int bufferSize, final Charset charset) {
		m_buffer = ByteBuffer.allocate(bufferSize);
		m_stringEncoder = charset
				.newEncoder()
				.onMalformedInput(CodingErrorAction.REPLACE)
				.onUnmappableCharacter(CodingErrorAction.REPLACE);
	}

	public byte[] data() {
		return m_buffer.array();
	}

	public int size() {
		return m_buffer.remaining();
	}

	private void reset() {
		m_buffer.rewind();
		m_buffer.limit(m_buffer.capacity());
		m_stringEncoder.reset();
	}

	public void encode(final CharSequence s) throws IOException {

		if (s.length() <= 0)
			return;

		final CharBuffer in = CharBuffer.wrap(s);

		if (s.length() <= m_buffer.capacity()) {

			reset();

			CoderResult cr = m_stringEncoder.encode(in, m_buffer, true);

			if (cr.isUnderflow())
				cr = m_stringEncoder.flush(m_buffer);

			if (cr.isOverflow())
				cr.throwException();

			m_buffer.flip();

		} else {
			throw new IOException("encode buffer size too small");
		}

	}

}
<commit_msg>Fixed possible string overflow in StringEncoder<commit_after>package se.culvertsoft.mgen.javapack.util;

import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
import java.nio.charset.Charset;
import java.nio.charset.CharsetEncoder;
import java.nio.charset.CoderResult;
import java.nio.charset.CodingErrorAction;

public class StringEncoder {

	private ByteBuffer m_buffer;

	private final CharsetEncoder m_stringEncoder;

	public StringEncoder(final int bufferSize, final Charset charset) {
		m_buffer = ByteBuffer.allocate(bufferSize);
		m_stringEncoder = charset
				.newEncoder()
				.onMalformedInput(CodingErrorAction.REPLACE)
				.onUnmappableCharacter(CodingErrorAction.REPLACE);
	}

	public byte[] data() {
		return m_buffer.array();
	}

	public int size() {
		return m_buffer.remaining();
	}

	private void reset() {
		m_buffer.rewind();
		m_buffer.limit(m_buffer.capacity());
		m_stringEncoder.reset();
	}

	public void encode(final CharSequence s) throws IOException {

		if (s.length() <= 0)
			return;

		final CharBuffer in = CharBuffer.wrap(s);

		if (s.length() > m_buffer.capacity())
			m_buffer = ByteBuffer.allocate(s.length());

		reset();

		CoderResult cr = m_stringEncoder.encode(in, m_buffer, true);

		if (cr.isUnderflow())
			cr = m_stringEncoder.flush(m_buffer);

		if (cr.isOverflow())
			cr.throwException();

		m_buffer.flip();

	}

}

{'fraction_non_alphanumeric': 0.11789924973204716, 'fraction_numerical': 0.0007145409074669524, 'mean_word_length': 4.511811023622047, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2280049', 'n_tokens_mistral': 1035, 'n_tokens_neox': 913, 'n_words': 192}