<commit_before>#ifndef CSV_H
#define CSV_H

#include <fstream>
#include <vector>
#include <map>

class CSV
{
public:
	CSV(std::string path);

	void open();
	bool readLine(std::vector<std::string> &items);
	long pos();
	long size();
	void close();

	enum Status { OK = 0, NotRead, Empty };

	Status status();

private:

	long _fileSize;
	long _filePosition;

	enum Encoding { Unknown = -1, UTF8 = 0, UTF16BE = 1, UTF16LE = 2, UTF32LE = 3, UTF32BE = 4 };

    Encoding _encoding;
    char _delim;

	bool readRaw();
	bool readUtf8();

	static const uint32_t NL = 0x0A;
	static const uint32_t CR = 0x0D;
	static const uint32_t TAB = 0x09;
	static const uint32_t SPACE = 0x20;
	static const uint32_t QUOTE = 0x22;
	static const uint32_t COMMA = 0x2C;
	static const uint32_t SEMICOLON = 0x3B;

	void determineEncoding();
	void determineDelimiters();

private:

	Status _status;

	int _rawBufferStartPos, _rawBufferEndPos;
	int _utf8BufferStartPos, _utf8BufferEndPos;
	std::string _path;
	std::ifstream _stream;
	bool _eof;

	char _rawBuffer[128];
	char _utf8Buffer[256];

	static inline bool utf16to8(char *out, char *in, int outSize, int inSize, int &written, int &read, bool bigEndian = false);
	static inline bool utf16to32(uint32_t &out, char *in, int inSize, int &bytesRead, bool bigEndian = false);
	static inline bool utf32to8(char *out, uint32_t in, int outSize, int &bytesWritten);
};

#endif // CSV_H<commit_msg>Increase CSV reader buffer size<commit_after>#ifndef CSV_H
#define CSV_H

#include <fstream>
#include <vector>
#include <map>

class CSV
{
public:
	CSV(std::string path);

	void open();
	bool readLine(std::vector<std::string> &items);
	long pos();
	long size();
	void close();

	enum Status { OK = 0, NotRead, Empty };

	Status status();

private:

	long _fileSize;
	long _filePosition;

	enum Encoding { Unknown = -1, UTF8 = 0, UTF16BE = 1, UTF16LE = 2, UTF32LE = 3, UTF32BE = 4 };

    Encoding _encoding;
    char _delim;

	bool readRaw();
	bool readUtf8();

	void determineEncoding();
	void determineDelimiters();

private:

	Status _status;

	int _rawBufferStartPos, _rawBufferEndPos;
	int _utf8BufferStartPos, _utf8BufferEndPos;
	std::string _path;
	std::ifstream _stream;
	bool _eof;

	char _rawBuffer[4096];
	char _utf8Buffer[8192];

	static inline bool utf16to8(char *out, char *in, int outSize, int inSize, int &written, int &read, bool bigEndian = false);
	static inline bool utf16to32(uint32_t &out, char *in, int inSize, int &bytesRead, bool bigEndian = false);
	static inline bool utf32to8(char *out, uint32_t in, int outSize, int &bytesWritten);
};

#endif // CSV_H
{'fraction_non_alphanumeric': 0.1140215716486903, 'fraction_numerical': 0.04352850539291217, 'mean_word_length': 4.758314855875832, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2436712', 'n_tokens_mistral': 1060, 'n_tokens_neox': 943, 'n_words': 296}