<commit_before>#include "stdafx.h"
#include "AnnFilesystem.hpp"
#include "AnnLogger.hpp"
#include "AnnGetter.hpp"

using namespace Annwvyn;
using namespace std;

vector<char> AnnFilesystemManager::charToEscape;
vector<char> AnnFilesystemManager::charToStrip;

AnnFileWriter::AnnFileWriter()
{
	AnnDebug() << "FileWriter instantiated";
}

void AnnFileWriter::write(shared_ptr<AnnSaveFileData> data)
{
	//Create the resources needed for the write operation
	auto fsmanager(AnnGetFileSystemManager());
	auto path(fsmanager->getPathForFileName(data->fileName));
	ofstream saveFile;

	//Make sure the "user save" directory as been created on the user's personal folder
	fsmanager->createSaveDirectory();

	//Open the file, abort if the file isn't openable
	saveFile.open(path); if (!saveFile.is_open())return;
	//push as plain text all key and data in a "key=data\n" format
	for (auto storedData : data->storedTextData)
		saveFile << storedData.first
		<< "="
		<< storedData.second
		<< endl;
	data->changed = false;
}

AnnFileReader::AnnFileReader()
{
	AnnDebug() << "FileReader instantiated";
}

AnnSaveFileDataPtr AnnFileReader::read(string fileName) const
{
	AnnDebug() << "Reading file " << fileName << " to memory";

	//Create the resource needed to the read operation
	auto fsmanager(AnnGetFileSystemManager());

	//Open the file
	ifstream ifile;
	auto fullPath = fsmanager->getPathForFileName(fileName);
	ifile.open(fullPath);

	if (!ifile)
	{
		AnnDebug() << "file " << fullPath << " doesn't exist or is not openable.";
		return nullptr;
	}

	auto fileData(fsmanager->crateSaveFileDataObject(fileName));
	if (!fileData) return nullptr;
	string buffer, key, value;

	//make sure the dataObject don't contain old content
	fileData->storedTextData.clear();

	//While we've not reach the end of the file
	while (!ifile.eof())
	{
		//Read a line
		getline(ifile, buffer);
		//Don't try to extract data from empty lines on the file
		if (buffer.empty()) continue;

		//Create a string stream on the buffer
		stringstream readStream(buffer);

		//Getline permit you to specify the "end-line" character. We use this to split the sting at the '=' in the file
		getline(readStream, key, '=');
		getline(readStream, value);

		//Store on the file data object the given key
		fileData->storedTextData[key] = value;
	}

	fileData->changed = false;
	return fileData;
}

AnnFilesystemManager::AnnFilesystemManager(string title) : AnnSubSystem("FilesystemManager"),
fileWriter(nullptr),
fileReader(nullptr)
{
	//get from the OS the user's personal directory
#ifdef WIN32
#pragma warning (disable : 4996) //Remove warning at usage of function "getenv"
	// ReSharper disable CppDeprecatedEntity
	pathToUserDir = getenv("USERPROFILE");
	// ReSharper restore CppDeprecatedEntity
#pragma warning (default : 4996)
#endif

#ifdef __linux__
	pathToUserDir = getenv("HOME");
#endif

	AnnDebug() << "Path got from operating system : " << pathToUserDir;

	//Forbidden characters in filename
	charToEscape.push_back(' ');
	charToEscape.push_back('&');
	charToEscape.push_back('<');
	charToEscape.push_back('>');
	charToEscape.push_back(':');
	charToEscape.push_back('|');
	charToEscape.push_back('?');
	charToEscape.push_back('*');
	charToEscape.push_back('"');
	charToEscape.push_back(':');
	charToEscape.push_back('%');

	//Forbidden characters in text "key=value" file
	charToStrip.push_back('=');
	charToStrip.push_back('\n');

	fileWriter = make_shared<AnnFileWriter>();
	fileReader = make_shared<AnnFileReader>();

	setSaveDirectoryName(title);
}

void AnnFilesystemManager::setSaveDirectoryName(string dirname)
{
	for (auto achar : charToEscape)
		replace(begin(dirname), end(dirname), achar, '_');
	saveDirectoryName = dirname;
#ifdef __linux__
	//Put the save directory as "hidden" in user's home directory on Linux.
	saveDirectoryName = "." + saveDirectoryName;
#endif

	AnnDebug() << "Save directory : " << saveDirectoryName;
	AnnDebug() << "Save directory location : " << getSaveDirectoryFullPath();
}

string AnnFilesystemManager::getPathForFileName(string filename) const
{
	if (!pathToUserDir.empty())
		return pathToUserDir + "/" + saveDirectoryName + "/" + filename;
	return "";
}

string AnnFilesystemManager::getSaveDirectoryFullPath() const
{
	return pathToUserDir + "/" + saveDirectoryName;
}

void AnnFilesystemManager::createDirectory(string path)
{
	//TODO clean that when upgrading to C++17
#ifdef WIN32
	//Win32 call to create a directory
	CreateDirectory(wstring{ begin(path), end(path) }.c_str(), nullptr);
#endif
#ifdef __linux__
	//POSIX call to create a directory
	auto status = mkdir(path.c_str(), S_IRWXU);
	if (status == 0 || status == EEXIST) return;
	AnnDebug() << "Warning: mkdir() did not return 0 or EEXIST.";
#endif
}

void AnnFilesystemManager::createSaveDirectory() const
{
	createDirectory(getSaveDirectoryFullPath());
}

void AnnFilesystemManager::releaseSaveFileDataObject(shared_ptr<AnnSaveFileData> data)
{
	cachedData.remove(data);
}

AnnSaveFileDataPtr AnnFilesystemManager::crateSaveFileDataObject(string filename)
{
	auto data = make_shared<AnnSaveFileData>(filename);
	cachedData.push_back(data);
	return data;
}<commit_msg>Add "ISSUE" in TODO comment<commit_after>#include "stdafx.h"
#include "AnnFilesystem.hpp"
#include "AnnLogger.hpp"
#include "AnnGetter.hpp"

using namespace Annwvyn;
using namespace std;

vector<char> AnnFilesystemManager::charToEscape;
vector<char> AnnFilesystemManager::charToStrip;

AnnFileWriter::AnnFileWriter()
{
	AnnDebug() << "FileWriter instantiated";
}

void AnnFileWriter::write(shared_ptr<AnnSaveFileData> data)
{
	//Create the resources needed for the write operation
	auto fsmanager(AnnGetFileSystemManager());
	auto path(fsmanager->getPathForFileName(data->fileName));
	ofstream saveFile;

	//Make sure the "user save" directory as been created on the user's personal folder
	fsmanager->createSaveDirectory();

	//Open the file, abort if the file isn't openable
	saveFile.open(path); if (!saveFile.is_open())return;
	//push as plain text all key and data in a "key=data\n" format
	for (auto storedData : data->storedTextData)
		saveFile << storedData.first
		<< "="
		<< storedData.second
		<< endl;
	data->changed = false;
}

AnnFileReader::AnnFileReader()
{
	AnnDebug() << "FileReader instantiated";
}

AnnSaveFileDataPtr AnnFileReader::read(string fileName) const
{
	AnnDebug() << "Reading file " << fileName << " to memory";

	//Create the resource needed to the read operation
	auto fsmanager(AnnGetFileSystemManager());

	//Open the file
	ifstream ifile;
	auto fullPath = fsmanager->getPathForFileName(fileName);
	ifile.open(fullPath);

	if (!ifile)
	{
		AnnDebug() << "file " << fullPath << " doesn't exist or is not openable.";
		return nullptr;
	}

	auto fileData(fsmanager->crateSaveFileDataObject(fileName));
	if (!fileData) return nullptr;
	string buffer, key, value;

	//make sure the dataObject don't contain old content
	fileData->storedTextData.clear();

	//While we've not reach the end of the file
	while (!ifile.eof())
	{
		//Read a line
		getline(ifile, buffer);
		//Don't try to extract data from empty lines on the file
		if (buffer.empty()) continue;

		//Create a string stream on the buffer
		stringstream readStream(buffer);

		//Getline permit you to specify the "end-line" character. We use this to split the sting at the '=' in the file
		getline(readStream, key, '=');
		getline(readStream, value);

		//Store on the file data object the given key
		fileData->storedTextData[key] = value;
	}

	fileData->changed = false;
	return fileData;
}

AnnFilesystemManager::AnnFilesystemManager(string title) : AnnSubSystem("FilesystemManager"),
fileWriter(nullptr),
fileReader(nullptr)
{
	//get from the OS the user's personal directory
#ifdef WIN32
#pragma warning (disable : 4996) //Remove warning at usage of function "getenv"
	// ReSharper disable CppDeprecatedEntity
	pathToUserDir = getenv("USERPROFILE");
	// ReSharper restore CppDeprecatedEntity
#pragma warning (default : 4996)
#endif

#ifdef __linux__
	pathToUserDir = getenv("HOME");
#endif

	AnnDebug() << "Path got from operating system : " << pathToUserDir;

	//Forbidden characters in filename
	charToEscape.push_back(' ');
	charToEscape.push_back('&');
	charToEscape.push_back('<');
	charToEscape.push_back('>');
	charToEscape.push_back(':');
	charToEscape.push_back('|');
	charToEscape.push_back('?');
	charToEscape.push_back('*');
	charToEscape.push_back('"');
	charToEscape.push_back(':');
	charToEscape.push_back('%');

	//Forbidden characters in text "key=value" file
	charToStrip.push_back('=');
	charToStrip.push_back('\n');

	fileWriter = make_shared<AnnFileWriter>();
	fileReader = make_shared<AnnFileReader>();

	setSaveDirectoryName(title);
}

void AnnFilesystemManager::setSaveDirectoryName(string dirname)
{
	for (auto achar : charToEscape)
		replace(begin(dirname), end(dirname), achar, '_');
	saveDirectoryName = dirname;
#ifdef __linux__
	//Put the save directory as "hidden" in user's home directory on Linux.
	saveDirectoryName = "." + saveDirectoryName;
#endif

	AnnDebug() << "Save directory : " << saveDirectoryName;
	AnnDebug() << "Save directory location : " << getSaveDirectoryFullPath();
}

string AnnFilesystemManager::getPathForFileName(string filename) const
{
	if (!pathToUserDir.empty())
		return pathToUserDir + "/" + saveDirectoryName + "/" + filename;
	return "";
}

string AnnFilesystemManager::getSaveDirectoryFullPath() const
{
	return pathToUserDir + "/" + saveDirectoryName;
}

void AnnFilesystemManager::createDirectory(string path)
{
	//TODO ISSUE clean that when upgrading to C++17
#ifdef WIN32
	//Win32 call to create a directory
	CreateDirectory(wstring{ begin(path), end(path) }.c_str(), nullptr);
#endif
#ifdef __linux__
	//POSIX call to create a directory
	auto status = mkdir(path.c_str(), S_IRWXU);
	if (status == 0 || status == EEXIST) return;
	AnnDebug() << "Warning: mkdir() did not return 0 or EEXIST.";
#endif
}

void AnnFilesystemManager::createSaveDirectory() const
{
	createDirectory(getSaveDirectoryFullPath());
}

void AnnFilesystemManager::releaseSaveFileDataObject(shared_ptr<AnnSaveFileData> data)
{
	cachedData.remove(data);
}

AnnSaveFileDataPtr AnnFilesystemManager::crateSaveFileDataObject(string filename)
{
	auto data = make_shared<AnnSaveFileData>(filename);
	cachedData.push_back(data);
	return data;
}
{'fraction_non_alphanumeric': 0.12705791206662792, 'fraction_numerical': 0.003486345148169669, 'mean_word_length': 5.628369704749679, 'pattern_counts': {'":': 0, '<': 95, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7210081', 'n_tokens_mistral': 3396, 'n_tokens_neox': 3290, 'n_words': 873}