<commit_before>//=====================================================================
//
// itoolbox.c - 工具函数大集合
//
// NOTE:
// for more information, please see the readme file.
//
//=====================================================================
#include "itoolbox.h"
#include "inetbase.h"
#include "imemdata.h"


//=====================================================================
// 工具函数
//=====================================================================

// 设置颜色：低4位是文字颜色，高4位是背景颜色
// 具体编码可以搜索 ansi color或者 
// http://en.wikipedia.org/wiki/ANSI_escape_code
void console_set_color(int color)
{
	#ifdef _WIN32
	HANDLE hConsole = GetStdHandle(STD_OUTPUT_HANDLE);
	WORD result = 0;
	if (color & 1) result |= FOREGROUND_RED;
	if (color & 2) result |= FOREGROUND_GREEN;
	if (color & 4) result |= FOREGROUND_BLUE;
	if (color & 8) result |= FOREGROUND_INTENSITY;
	if (color & 16) result |= BACKGROUND_RED;
	if (color & 32) result |= BACKGROUND_GREEN;
	if (color & 64) result |= BACKGROUND_BLUE;
	if (color & 128) result |= BACKGROUND_INTENSITY;
	SetConsoleTextAttribute(hConsole, (WORD)result);
	#else
	int foreground = color & 7;
	int background = (color >> 4) & 7;
	int bold = color & 8;
	printf("\033[%s3%d;4%dm", bold? "01;" : "", foreground, background);
	#endif
}

// 设置光标位置左上角是，行与列都是从1开始计数的
void console_cursor(int row, int col)
{
	#ifdef _WIN32
	COORD point; 
	point.X = col - 1;
	point.Y = row - 1;
	SetConsoleCursorPosition(GetStdHandle(STD_OUTPUT_HANDLE), point); 
	#else
	printf("\033[%d;%dH", row, col);
	#endif
}

// 恢复屏幕颜色
void console_reset(void)
{
	#ifdef _WIN32
	console_set_color(7);
	#else
	printf("\033[0m");
	#endif
}

// 清屏
void console_clear(int color)
{
	#ifdef _WIN32
	COORD coordScreen = { 0, 0 };
	DWORD cCharsWritten;
	CONSOLE_SCREEN_BUFFER_INFO csbi;
	DWORD dwConSize;
	HANDLE hConsole = GetStdHandle(STD_OUTPUT_HANDLE);
	GetConsoleScreenBufferInfo(hConsole, &csbi);
	dwConSize = csbi.dwSize.X * csbi.dwSize.Y;
	FillConsoleOutputCharacter(hConsole, TEXT(' '),
								dwConSize,
								coordScreen,
								&cCharsWritten);
	GetConsoleScreenBufferInfo(hConsole, &csbi);
	FillConsoleOutputAttribute(hConsole,
								csbi.wAttributes,
								dwConSize,
								coordScreen,
								&cCharsWritten);
	SetConsoleCursorPosition(hConsole, coordScreen); 
	#else
	printf("\033[2J");
	#endif
}




//=====================================================================
// CSV Reader/Writer
//=====================================================================
struct iCsvReader
{<commit_msg>commit new toolbox<commit_after>//=====================================================================
//
// itoolbox.c - 工具函数大集合
//
// NOTE:
// for more information, please see the readme file.
//
//=====================================================================
#include "itoolbox.h"
#include "inetbase.h"
#include "imemdata.h"


//=====================================================================
// 工具函数
//=====================================================================

// 设置颜色：低4位是文字颜色，高4位是背景颜色
// 具体编码可以搜索 ansi color或者 
// http://en.wikipedia.org/wiki/ANSI_escape_code
void console_set_color(int color)
{
	#ifdef _WIN32
	HANDLE hConsole = GetStdHandle(STD_OUTPUT_HANDLE);
	WORD result = 0;
	if (color & 1) result |= FOREGROUND_RED;
	if (color & 2) result |= FOREGROUND_GREEN;
	if (color & 4) result |= FOREGROUND_BLUE;
	if (color & 8) result |= FOREGROUND_INTENSITY;
	if (color & 16) result |= BACKGROUND_RED;
	if (color & 32) result |= BACKGROUND_GREEN;
	if (color & 64) result |= BACKGROUND_BLUE;
	if (color & 128) result |= BACKGROUND_INTENSITY;
	SetConsoleTextAttribute(hConsole, (WORD)result);
	#else
	int foreground = color & 7;
	int background = (color >> 4) & 7;
	int bold = color & 8;
	printf("\033[%s3%d;4%dm", bold? "01;" : "", foreground, background);
	#endif
}

// 设置光标位置左上角是，行与列都是从1开始计数的
void console_cursor(int row, int col)
{
	#ifdef _WIN32
	COORD point; 
	point.X = col - 1;
	point.Y = row - 1;
	SetConsoleCursorPosition(GetStdHandle(STD_OUTPUT_HANDLE), point); 
	#else
	printf("\033[%d;%dH", row, col);
	#endif
}

// 恢复屏幕颜色
void console_reset(void)
{
	#ifdef _WIN32
	console_set_color(7);
	#else
	printf("\033[0m");
	#endif
}

// 清屏
void console_clear(int color)
{
	#ifdef _WIN32
	COORD coordScreen = { 0, 0 };
	DWORD cCharsWritten;
	CONSOLE_SCREEN_BUFFER_INFO csbi;
	DWORD dwConSize;
	HANDLE hConsole = GetStdHandle(STD_OUTPUT_HANDLE);
	GetConsoleScreenBufferInfo(hConsole, &csbi);
	dwConSize = csbi.dwSize.X * csbi.dwSize.Y;
	FillConsoleOutputCharacter(hConsole, TEXT(' '),
								dwConSize,
								coordScreen,
								&cCharsWritten);
	GetConsoleScreenBufferInfo(hConsole, &csbi);
	FillConsoleOutputAttribute(hConsole,
								csbi.wAttributes,
								dwConSize,
								coordScreen,
								&cCharsWritten);
	SetConsoleCursorPosition(hConsole, coordScreen); 
	#else
	printf("\033[2J");
	#endif
}



#ifndef IPOSIX_STACK_BUFFER_SIZE
#define IPOSIX_STACK_BUFFER_SIZE	1024
#endif

#ifndef IDISABLE_FILE_SYSTEM_ACCESS

/* load line: returns -1 for end of file, 0 for success */
int iposix_file_read_line(FILE *fp, ivalue_t *str)
{
	const int bufsize = IPOSIX_STACK_BUFFER_SIZE;
	int size, eof = 0;
	char buffer[IPOSIX_STACK_BUFFER_SIZE];

	it_sresize(str, 0);
	for (size = 0, eof = 0; ; ) {
		int ch = fgetc(fp);
		if (ch < 0) {
			eof = 1;
			break;
		}
		buffer[size++] = (unsigned char)ch;
		if (size >= bufsize) {
			it_strcatc(str, buffer, size);
			size = 0;
		}
		if (ch == '\n') break;
	}
	if (size > 0) {
		it_strcatc(str, buffer, size);
	}
	if (eof && it_size(str) == 0) return -1;
	it_strstripc(str, "\r\n");
	return 0;
}

#endif


//=====================================================================
// CSV Reader/Writer
//=====================================================================
struct iCsvReader
{
{'fraction_non_alphanumeric': 0.2626680455015512, 'fraction_numerical': 0.020854877628404, 'mean_word_length': 4.756944444444445, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2834918', 'n_tokens_mistral': 2348, 'n_tokens_neox': 2079, 'n_words': 432}