<commit_before>#include <Document/ProjectManager.hh>
#include <Document/wordUnit.hh>
#include <QDir>
#include <QVector>
#include <QTextStream>
#include <QStandardPaths>
#include <QCoreApplication>
#include <QFileInfo>
#include <QProcess>
#include <iostream>

ProjectManager::ProjectManager(QString projDirName){
  loadEmptyDefault(projDirName);
}

void ProjectManager::loadEmptyDefault(QString projDirName){
  QString designatedRoot;

  designatedRoot = QStandardPaths::standardLocations(QStandardPaths::DocumentsLocation)[0] +
                                                              "/decipher_text Projects";

  configureRoot(designatedRoot, projDirName);

  verifyProjectRoot();

  createEmptyProject();

  //Should use direct set to avoid calling event manager too soon.
  saveState = false;
  saveHistory = false;

  setCurrentPageDefault();
  setSaveFileName("Untitled Project");
  setSavedUrl(getWorkingDir() + "/" + "Untitled Project.dtp");
}

void ProjectManager::configureRoot(QString rootName, QString projDirName){
  rootDir = rootName;
  dirName = projDirName;
  projectName = projDirName;
}

void ProjectManager::setSaveState(bool staat){
  if(!getSaveHistory() && staat == true) setSaveHistory(true);
  saveState = staat;

  publishSave();
}

bool ProjectManager::getSaveState(){
  return saveState;
}

void ProjectManager::setSaveHistory(bool staat){
  saveHistory = staat;
}

void ProjectManager::setSavedUrl(QString urlData){
  savedUrl = urlData;
}

QString ProjectManager::getSavedUrl(){
  return savedUrl;
}

bool ProjectManager::getSaveHistory(){
  return saveHistory;
}

void ProjectManager::verifyProjectRoot(){
  QDir rootVerify;
  rootVerify.setPath(this->rootDir);

  if (!rootVerify.exists()){
    rootVerify.mkdir(this->rootDir);
  }
}

QList<Page>* ProjectManager::emitPages(){
  return &pageList;
}

void ProjectManager::setCurrentPage(int pgIndex){
  currentPage = pgIndex;
}

int ProjectManager::getCurrentPage(){
  return currentPage;
}

void ProjectManager::setCurrentPageDefault(){
  setCurrentPage(-1);
}

void ProjectManager::addPages(QStringList pages){
  int i, lim;
  lim = pages.length();

  for(i = 0; i < lim; i++){
    addSinglePage(pages[i]);
  }

  publishPagesChanged();
}

void ProjectManager::addPages(QString page){
  addSinglePage(page);
  publishPagesChanged();
}

void ProjectManager::addSinglePage(QString page){
  QFileInfo oldFileInfo(page);
  QString ofName, imagesRoot, imagesFull, imagesRefined, imagesThumb, commandExe;
  int futurePageIndex;

  ofName = oldFileInfo.fileName();

  imagesRoot = getWorkingDir() + "/images";
  imagesFull = imagesRoot + "/full";
  imagesRefined = imagesRoot + "/refined";
  imagesThumb = imagesRoot + "/thumbnail";

  QFile::copy(page, imagesFull + "/" + ofName);

  commandExe =  "\"" + getImageMagickCommand() + "\" ";
  commandExe += "\"" + imagesFull + "/" + ofName + "\"";
  commandExe += " -thumbnail 104x148! ";
  commandExe += "\"" + imagesThumb + "/" + ofName + "\"";

  QProcess::execute(commandExe);

  futurePageIndex = pageList.length();

  commitPage(ofName, futurePageIndex);
}

void ProjectManager::commitPage(QString ofName, int index){
  Page dummyPage(getWorkingDir(), index);
  dummyPage.setFileName(ofName);

  pageList.push_back(dummyPage);
  setSaveState(false);
}

void ProjectManager::createEmptyProject(){
  QString targetName, root, dirName;
  QDir targetDir;

  targetName = getWorkingDir();
  targetDir.setPath(targetName);

  //Remove old contents if exists
  if (targetDir.exists()){
    removeDir(targetName);
  }

  targetDir.mkdir(targetName);
  createProjectFile(getWorkingDir() + "/Untitled Project.dtp", "");

  createImageDirs();
}

void ProjectManager::createImageDirs(){
  QString projectWorkingDir, imageRootDir;
  QDir dummyDir;

  projectWorkingDir = getWorkingDir();
  imageRootDir = projectWorkingDir + "/images";

  dummyDir.mkdir(imageRootDir);
  dummyDir.mkdir(imageRootDir + "/full");
  dummyDir.mkdir(imageRootDir + "/refined");
  dummyDir.mkdir(imageRootDir + "/thumbnail");
}

void ProjectManager::createProjectFile(QString targetName, QString projContents){
  QFile pFile;


  pFile.setFileName(targetName);

  if (pFile.open(QIODevice::WriteOnly)){
    QTextStream out(&pFile);

    out << projContents;

    pFile.close();
  }<commit_msg>Solved the issue with loading and saving unicode files on windows.<commit_after>#include <Document/ProjectManager.hh>
#include <Document/wordUnit.hh>
#include <QDir>
#include <QVector>
#include <QTextStream>
#include <QStandardPaths>
#include <QCoreApplication>
#include <QFileInfo>
#include <QProcess>
#include <iostream>

ProjectManager::ProjectManager(QString projDirName){
  loadEmptyDefault(projDirName);
}

void ProjectManager::loadEmptyDefault(QString projDirName){
  QString designatedRoot;

  designatedRoot = QStandardPaths::standardLocations(QStandardPaths::DocumentsLocation)[0] +
                                                              "/decipher_text Projects";

  configureRoot(designatedRoot, projDirName);

  verifyProjectRoot();

  createEmptyProject();

  //Should use direct set to avoid calling event manager too soon.
  saveState = false;
  saveHistory = false;

  setCurrentPageDefault();
  setSaveFileName("Untitled Project");
  setSavedUrl(getWorkingDir() + "/" + "Untitled Project.dtp");
}

void ProjectManager::configureRoot(QString rootName, QString projDirName){
  rootDir = rootName;
  dirName = projDirName;
  projectName = projDirName;
}

void ProjectManager::setSaveState(bool staat){
  if(!getSaveHistory() && staat == true) setSaveHistory(true);
  saveState = staat;

  publishSave();
}

bool ProjectManager::getSaveState(){
  return saveState;
}

void ProjectManager::setSaveHistory(bool staat){
  saveHistory = staat;
}

void ProjectManager::setSavedUrl(QString urlData){
  savedUrl = urlData;
}

QString ProjectManager::getSavedUrl(){
  return savedUrl;
}

bool ProjectManager::getSaveHistory(){
  return saveHistory;
}

void ProjectManager::verifyProjectRoot(){
  QDir rootVerify;
  rootVerify.setPath(this->rootDir);

  if (!rootVerify.exists()){
    rootVerify.mkdir(this->rootDir);
  }
}

QList<Page>* ProjectManager::emitPages(){
  return &pageList;
}

void ProjectManager::setCurrentPage(int pgIndex){
  currentPage = pgIndex;
}

int ProjectManager::getCurrentPage(){
  return currentPage;
}

void ProjectManager::setCurrentPageDefault(){
  setCurrentPage(-1);
}

void ProjectManager::addPages(QStringList pages){
  int i, lim;
  lim = pages.length();

  for(i = 0; i < lim; i++){
    addSinglePage(pages[i]);
  }

  publishPagesChanged();
}

void ProjectManager::addPages(QString page){
  addSinglePage(page);
  publishPagesChanged();
}

void ProjectManager::addSinglePage(QString page){
  QFileInfo oldFileInfo(page);
  QString ofName, imagesRoot, imagesFull, imagesRefined, imagesThumb, commandExe;
  int futurePageIndex;

  ofName = oldFileInfo.fileName();

  imagesRoot = getWorkingDir() + "/images";
  imagesFull = imagesRoot + "/full";
  imagesRefined = imagesRoot + "/refined";
  imagesThumb = imagesRoot + "/thumbnail";

  QFile::copy(page, imagesFull + "/" + ofName);

  commandExe =  "\"" + getImageMagickCommand() + "\" ";
  commandExe += "\"" + imagesFull + "/" + ofName + "\"";
  commandExe += " -thumbnail 104x148! ";
  commandExe += "\"" + imagesThumb + "/" + ofName + "\"";

  QProcess::execute(commandExe);

  futurePageIndex = pageList.length();

  commitPage(ofName, futurePageIndex);
}

void ProjectManager::commitPage(QString ofName, int index){
  Page dummyPage(getWorkingDir(), index);
  dummyPage.setFileName(ofName);

  pageList.push_back(dummyPage);
  setSaveState(false);
}

void ProjectManager::createEmptyProject(){
  QString targetName, root, dirName;
  QDir targetDir;

  targetName = getWorkingDir();
  targetDir.setPath(targetName);

  //Remove old contents if exists
  if (targetDir.exists()){
    removeDir(targetName);
  }

  targetDir.mkdir(targetName);
  createProjectFile(getWorkingDir() + "/Untitled Project.dtp", "");

  createImageDirs();
}

void ProjectManager::createImageDirs(){
  QString projectWorkingDir, imageRootDir;
  QDir dummyDir;

  projectWorkingDir = getWorkingDir();
  imageRootDir = projectWorkingDir + "/images";

  dummyDir.mkdir(imageRootDir);
  dummyDir.mkdir(imageRootDir + "/full");
  dummyDir.mkdir(imageRootDir + "/refined");
  dummyDir.mkdir(imageRootDir + "/thumbnail");
}

void ProjectManager::createProjectFile(QString targetName, QString projContents){
  QFile pFile;


  pFile.setFileName(targetName);

  if (pFile.open(QIODevice::WriteOnly | QIODevice::Text)){
    QString newContents;
    QTextStream out(&pFile);
    out.setCodec("UTF-8");
    out.setAutoDetectUnicode(true);
    out.setGenerateByteOrderMark(true);

    newContents = QString::fromUtf8(projContents.toUtf8().data());

    out << projContents;

    pFile.close();
  }
{'fraction_non_alphanumeric': 0.13128460061010055, 'fraction_numerical': 0.0023726132640379616, 'mean_word_length': 5.10904071773637, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6864754', 'n_tokens_mistral': 2842, 'n_tokens_neox': 2765, 'n_words': 538}