<commit_before>﻿#include "rdi_wfileio.h"
#include "utf8_file_dir.h"

#include <sstream>
#include <fstream>
#include <codecvt>
#include <utility>
#include <assert.h>
#include <algorithm>
#include <memory>

using namespace std;

namespace RDI
{

wstring read_wfile(const string& filename)
{
	wifstream wif(filename.c_str());
	wif.imbue(locale(wif.getloc(), new codecvt_utf8<wchar_t>));
	wstringstream wss;
	wss << wif.rdbuf();
	return wss.str();
}

bool write_wfile(const string& filename, const wstring& fileContent)
{
	wofstream wof(filename.c_str());
	if(!wof.is_open())
		return false;
	wof.imbue(locale(wof.getloc(), new codecvt_utf8<wchar_t>));
	wof << fileContent;
	return true;
}

string read_file(const string &filename)
{
	ifstream stream(filename.c_str());
	stringstream ss;
	ss << stream.rdbuf();
	return ss.str();
}

bool write_file(const string &filename, const string &fileContent)
{
	ofstream of(filename.c_str());
	if(!of.is_open())
		return false;
	of << fileContent;
	return true;
}

string get_current_directory()
{
	char* strPath = new char[512];
	CubicleSoft::UTF8::Dir::Getcwd(strPath, 512);

	string path(strPath);
	delete[] strPath;
	return path + "/";
}

vector<wstring> read_wfile_lines(const string &filename)
{
	wifstream wif(filename.c_str());
	wif.imbue(locale(wif.getloc(), new codecvt_utf8<wchar_t>));
	wstringstream wss;
	wss << wif.rdbuf();

	wstring tmp;
	vector<wstring> lines;
	while (getline(wss, tmp))
		lines.push_back(tmp);

	return lines;
}

bool write_wfile_lines(const string &filename, const vector<wstring> &linesToWrite)
{
	wofstream wof(filename.c_str());<commit_msg>setting LC_ALL to "en_US.UTF-8"<commit_after>﻿#include "rdi_wfileio.h"
#include "utf8_file_dir.h"

#include <sstream>
#include <fstream>
#include <codecvt>
#include <utility>
#include <assert.h>
#include <algorithm>
#include <memory>

using namespace std;

namespace RDI
{

wstring read_wfile(const string& filename)
{
	wifstream wif(filename.c_str());
	wif.imbue(locale(wif.getloc(), new codecvt_utf8<wchar_t>));
	wstringstream wss;
	wss << wif.rdbuf();
	return wss.str();
}

bool write_wfile(const string& filename, const wstring& fileContent)
{
	wofstream wof(filename.c_str());
	if(!wof.is_open())
		return false;
	wof.imbue(locale(wof.getloc(), new codecvt_utf8<wchar_t>));
	wof << fileContent;
	return true;
}

string read_file(const string &filename)
{
	ifstream stream(filename.c_str());
	stringstream ss;
	ss << stream.rdbuf();
	return ss.str();
}

bool write_file(const string &filename, const string &fileContent)
{
	ofstream of(filename.c_str());
	if(!of.is_open())
		return false;
	of << fileContent;
	return true;
}

string get_current_directory()
{
	char* strPath = new char[512];
	CubicleSoft::UTF8::Dir::Getcwd(strPath, 512);

	string path(strPath);
	delete[] strPath;
	return path + "/";
}

vector<wstring> read_wfile_lines(const string &filename)
{
	std::setlocale(LC_ALL, "en_US.UTF-8");
	wifstream wif(filename.c_str());
	wif.imbue(locale(wif.getloc(), new codecvt_utf8<wchar_t>));
	wstringstream wss;
	wss << wif.rdbuf();

	wstring tmp;
	vector<wstring> lines;
	while (getline(wss, tmp))
		lines.push_back(tmp);

	return lines;
}

bool write_wfile_lines(const string &filename, const vector<wstring> &linesToWrite)
{
	wofstream wof(filename.c_str());
{'fraction_non_alphanumeric': 0.15155203895313452, 'fraction_numerical': 0.007303712720632989, 'mean_word_length': 6.068817204301075, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2425341', 'n_tokens_mistral': 1319, 'n_tokens_neox': 1303, 'n_words': 266}