<commit_before>#define DEBUG 0

const uint16_t MAX_BUFFER_SIZE = SERIAL_RX_BUFFER_SIZE - 1;
//const uint32_t BLOCK_SIZE = 32 * 1024;
const uint32_t BLOCK_SIZE = 2000 * 1024;
//uint16_t buffer_size = 0
uint16_t buffer_size = 240;
uint32_t threshold = 0;

void message(const __FlashStringHelper *msg)
{
	Serial.print(F("M"));
	Serial.println(msg);
}

void setup()
{
	Serial.begin(115200);
	while (!Serial) {
		; // wait for serial port to connect. Needed for Leonardo only
	}
	Serial.print(F("!Maximum serial buffer size: "));
	Serial.println(MAX_BUFFER_SIZE);
}

void loop()
{
	// Flush
	Serial.flush();
	while (Serial.available() > 0) {
#if DEBUG
		Serial.print(F("!available() = "));
		Serial.println(Serial.available());
#endif
		Serial.read();
	}
	//
	char send_buffer_first[6];
	char send_buffer[6];
	const char *fmt = "S%4d";
	buffer_size++;
	if (buffer_size > MAX_BUFFER_SIZE) {
		buffer_size = MAX_BUFFER_SIZE;
	}
	snprintf(send_buffer_first, sizeof send_buffer_first, fmt, buffer_size);
	// Choice of threshold
	threshold = buffer_size * 3 / 4 + 1;
	//threshold = buffer_size;
	snprintf(send_buffer, sizeof send_buffer, fmt, threshold);
	//
	int c = -1;
	uint32_t count = 0;
	uint32_t n = BLOCK_SIZE;
	uint32_t current_threshold = threshold;
	Serial.print(F("!Buffer size: "));
	Serial.print(buffer_size);
	Serial.print(F(", threshold: "));
	Serial.println(threshold);
#if DEBUG
	Serial.print(F("!send_buffer_first="));
	Serial.println(send_buffer_first);
	Serial.print(F("!send_buffer="));
	Serial.println(send_buffer);
#endif
	//
	Serial.println(F("R"));
	Serial.println(send_buffer_first);
	while (n) {
		int a;
		while (n && (a = Serial.available()) > 0) {
			c = Serial.read();
#if DEBUG
			Serial.print(F("Dc = "));
			Serial.print(c);
			Serial.print(F(", "));
			Serial.print(char(c));
			Serial.print(F(", a="));
			Serial.println(a);<commit_msg>Fix the 16 bit overflow on BLOCK_SIZE<commit_after>#define DEBUG 0

const uint16_t MAX_BUFFER_SIZE = SERIAL_RX_BUFFER_SIZE - 1;
const uint32_t BLOCK_SIZE = 32 * 1024L;
uint16_t buffer_size = 0;
uint32_t threshold = 0;

void setup()
{
	Serial.begin(115200);
	while (!Serial) {
		; // wait for serial port to connect. Needed for Leonardo only
	}
	Serial.print(F("!Maximum serial buffer size: "));
	Serial.println(MAX_BUFFER_SIZE);
}

void loop()
{
	// Flush the serial input buffer
	Serial.flush();
	while (Serial.available() > 0) {
#if DEBUG
		Serial.print(F("!available() = "));
		Serial.println(Serial.available());
#endif
		Serial.read();
	}
	//
	char send_buffer_first[6];
	char send_buffer[6];
	const char *fmt = "S%4d";
	buffer_size++;
	if (buffer_size > MAX_BUFFER_SIZE) {
		buffer_size = MAX_BUFFER_SIZE;
	}
	snprintf(send_buffer_first, sizeof send_buffer_first, fmt, buffer_size);
	// Choice of threshold
	//threshold = buffer_size * 3 / 4 + 1;
	threshold = buffer_size;
	snprintf(send_buffer, sizeof send_buffer, fmt, threshold);
	//
	int c = -1;
	uint32_t count = 0;
	uint32_t n = BLOCK_SIZE;
	uint32_t current_threshold = threshold;
	Serial.print(F("!Buffer size: "));
	Serial.print(buffer_size);
	Serial.print(F(", threshold: "));
	Serial.println(threshold);
#if DEBUG
	Serial.print(F("!send_buffer_first="));
	Serial.println(send_buffer_first);
	Serial.print(F("!send_buffer="));
	Serial.println(send_buffer);
#endif
	//
	Serial.println(F("R"));
	Serial.println(send_buffer_first);
	while (n) {
		int a;
		while (n && (a = Serial.available()) > 0) {
#if DEBUG
			Serial.print(F("Dn="));
			Serial.println(n);
#endif
			c = Serial.read();
			// Process the data
			delayMicroseconds(50);
#if DEBUG
			Serial.print(F("Dc = "));
			Serial.print(c);
			Serial.print(F(", "));
			Serial.print(char(c));
			Serial.print(F(", a="));
			Serial.println(a);
{'fraction_non_alphanumeric': 0.16304055868922912, 'fraction_numerical': 0.0265914585012087, 'mean_word_length': 4.566517189835576, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4467283', 'n_tokens_mistral': 1506, 'n_tokens_neox': 1381, 'n_words': 298}