<commit_before>/*
 *    Copyright 2002-2013 CEA LIST
 * 
 *    This file is part of LIMA.
 * 
 *    LIMA is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Affero General Public License as published by
 *    the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 * 
 *    LIMA is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 * 
 *    You should have received a copy of the GNU Affero General Public License
 *    along with LIMA.  If not, see <http://www.gnu.org/licenses/>
 */
#include "ResourceTool.h"
#include "ResourceEditorTableModel.h"
#include "ResourceFileEditDlg.h"
#include "DictionnaryEntry.h"
#include "DictionnaryEntryEditDlg.h"
#include "NGramEntry.h"
#include "SyntacticRuleEntry.h"
#include "IdiomaticExpressionEntry.h"
#include "IdiomaticExpressionEntryEditDlg.h"


#include <QtXml/QDomImplementation>
#include <QtXml/QDomDocument>
#include <QtCore/QtDebug>
#include <QtCore/QSettings>
#include <QtCore/QSharedMemory>
#include <QtCore/QTimer>
#include <QtCore/QFile>
#include <QtGui/QMainWindow>


using namespace Lima::Pelf;

ResourceTool::ResourceTool (QWidget* p):
    QMainWindow(p),
    Ui::PelfResourceToolWindow(),
    linguisticDataRoot(""),
    commandsRoot(""),
    evaluationStatus(EVALUATION_READY)
{
    setupUi(this);
}

bool ResourceTool::init ()
{
    qDebug() << "Starting application, loading settings...";
    settings = new QSettings("CEA LIST LIMA", "Pelf Resource Tool");
    restoreGeometry(settings->value("geometry").toByteArray());

    char* mmdist = getenv("LIMA_DIST");
    if (mmdist == 0) mmdist = "/usr";
    if(!xmlFileInit(QString(mmdist)+"/share/config/pelf/pelf-rt-conf.xml"))
    {
      qDebug() << "Unable to load configuration from file "<<(QString(mmdist)+"/share/config/pelf/pelf-rt-conf.xml")<<", aborting";
      return false;
    }

    initPelfSharedMemory();
    if(!dictionnaryResourcePath.isEmpty() && !dictionnaryInstallCommand.isEmpty())
        dictionnaryInit(dictionnaryResourcePath, dictionnaryInstallCommand);
    else
        editorTabsWidget->removeTab(editorTabsWidget->indexOf(dictionnaryTab));
    if(!ngramsResourcePaths.isEmpty() && !ngramsInstallCommand.isEmpty())
    {
        nGramsInit(ngramsResourcePaths, ngramsInstallCommand);
        if(!ngramsSymbolicCodesPath.isEmpty() && !ngramsCorpusPath.isEmpty())
            nGramsCorpusInit(ngramsSymbolicCodesPath, ngramsCorpusPath);
    }
    else
        editorTabsWidget->removeTab(editorTabsWidget->indexOf(nGramsTab));
    if(!syntacticRulesResourcePaths.isEmpty() && !syntacticRulesInstallCommand.isEmpty())
        syntacticRulesInit(syntacticRulesResourcePaths, syntacticRulesInstallCommand);
    else
        editorTabsWidget->removeTab(editorTabsWidget->indexOf(syntacticRulesTab));
    if(!idiomaticExpressionsResourcePath.isEmpty() && !idiomaticExpressionsInstallCommand.isEmpty())
        idiomaticExpressionsInit(idiomaticExpressionsResourcePath, idiomaticExpressionsInstallCommand);
    else
        editorTabsWidget->removeTab(editorTabsWidget->indexOf(idiomaticExpressionsTab));

    connect(evaluationBtn, SIGNAL(clicked(bool)), this, SLOT(confirmEvaluate()));

    messagesSplitter->restoreState(settings->value ("messagesSplitter").toByteArray());
    ngramsCategoriesSplitter->restoreState(settings->value ("ngramsCategoriesSplitter").toByteArray());
    ngramsOccurencesSplitter->restoreState(settings->value ("ngramsOccurencesSplitter").toByteArray());

    
    show();
    return true;
}

bool ResourceTool::xmlFileInit (QString xmlFilePath)
{
    QFile xmlFile(xmlFilePath);
    if(!xmlFile.open(QFile::ReadOnly)){
        qDebug() << "Unable to open xml configuration file, aborting";
        return false;
    }
    QDomDocument doc;
    QString errorMsg;
    if(!doc.setContent(&xmlFile, false, &errorMsg))
    {
        qDebug() << "Unable to parse xml file, aborting (" << errorMsg << ")";
        xmlFile.close();
        return false;
    }
    xmlFile.close();
    QDomElement configurationNode = doc.documentElement();
    if(configurationNode.tagName() != "configuration" )
    {
        qDebug() << "Unrecognized xml file format, aborting";
        return false;
    }

    QDomElement linguisticDataRootNode = configurationNode.firstChildElement("linguisticDataRoot");
    if(!linguisticDataRootNode.isNull() && !linguisticDataRootNode.attribute("path").isNull())
        linguisticDataRoot = linguisticDataRootNode.attribute("path");
    if(linguisticDataRoot.isEmpty())
        linguisticDataRoot = getenv("LINGUISTIC_DATA_ROOT");
    if(linguisticDataRoot.isEmpty())
    {
      qDebug() << "Linguistic data root is not defined";
      return false;
    }
    setenv("LINGUISTIC_DATA_ROOT", linguisticDataRoot.toUtf8(), 1);
    qDebug() << "Retrieved and set env variable linguistic data root to " << linguisticDataRoot;

    QDomElement commandsRootNode = configurationNode.firstChildElement("commandsRoot");
    if(!commandsRootNode.isNull() && !commandsRootNode.attribute("path").isNull())
        commandsRoot = commandsRootNode.attribute("path");
    if(commandsRoot.isEmpty())
    {
      commandsRoot = getenv("LIMA_DIST");
      if (commandsRoot.isEmpty())
      {
        commandsRoot = QString("/usr");
      }
    }
    commandsRoot += "/share/apps/pelf/commands";
    setenv("PELF_COMMANDS_ROOT", commandsRoot.toUtf8(), 1);
    qDebug() << "Retrieved command root " << commandsRoot;

    QDomElement resourcesNode = configurationNode.firstChildElement("resources");
    if(!resourcesNode.isNull())
    {
        QDomElement dictionnaryNode = resourcesNode.firstChildElement("dictionnary");
        if(!dictionnaryNode.isNull())
        {
            QDomElement resourceFileNode = dictionnaryNode.firstChildElement("resourceFile");
            if(!resourceFileNode.isNull() && !resourceFileNode.attribute("pathSuffix").isNull())
            {
                dictionnaryResourcePath = resourceFileNode.attribute("pathSuffix");
                if (!dictionnaryResourcePath.startsWith('/'))
                  dictionnaryResourcePath.prepend(linguisticDataRoot+"/");
            }
            QDomElement installCommandFileNode = dictionnaryNode.firstChildElement("installCommandFile");
            if(!installCommandFileNode.isNull() && !installCommandFileNode.attribute("pathSuffix").isNull())
                dictionnaryInstallCommand = commandsRoot+"/"+installCommandFileNode.attribute("pathSuffix");
        }
        QDomElement ngramsNode = resourcesNode.firstChildElement("ngrams");
        if(!ngramsNode.isNull())
        {
            QDomElement resourceFileNode = ngramsNode.firstChildElement("resourceFile");
            while(!resourceFileNode.isNull())
            {
                if(!resourceFileNode.isNull() && !resourceFileNode.attribute("pathSuffix").isNull())
								{
									if (resourceFileNode.attribute("pathSuffix").startsWith('/'))
                    ngramsResourcePaths << resourceFileNode.attribute("pathSuffix");
									else
										ngramsResourcePaths << linguisticDataRoot+"/"+resourceFileNode.attribute("pathSuffix");
									resourceFileNode = resourceFileNode.nextSiblingElement("resourceFile");
								}
            }
            QDomElement installCommandFileNode = ngramsNode.firstChildElement("installCommandFile");
            if(!installCommandFileNode.isNull() && !installCommandFileNode.attribute("pathSuffix").isNull())
                ngramsInstallCommand = commandsRoot+"/"+installCommandFileNode.attribute("pathSuffix");
            QDomElement symbolicCodeFileNode = ngramsNode.firstChildElement("symbolicCodeFile");
            if(!symbolicCodeFileNode.isNull() && !symbolicCodeFileNode.attribute("pathSuffix").isNull())
                ngramsSymbolicCodesPath = linguisticDataRoot+"/"+symbolicCodeFileNode.attribute("pathSuffix");
            QDomElement corpusFileNode = ngramsNode.firstChildElement("corpusFile");
            if(!corpusFileNode.isNull() && !corpusFileNode.attribute("pathSuffix").isNull())
            {
              ngramsCorpusPath = corpusFileNode.attribute("pathSuffix");
              if (!ngramsCorpusPath.startsWith('/'))
                ngramsCorpusPath.prepend(linguisticDataRoot+"/");
            }
        }
        QDomElement syntacticRulesNode = resourcesNode.firstChildElement("syntacticRules");
        if(!syntacticRulesNode.isNull())
        {
            QDomElement resourceFileNode = syntacticRulesNode.firstChildElement("resourceFile");
            while(!resourceFileNode.isNull())
            {
                if(!resourceFileNode.isNull() && !resourceFileNode.attribute("pathSuffix").isNull())
                    syntacticRulesResourcePaths << linguisticDataRoot+"/"+resourceFileNode.attribute("pathSuffix");
                resourceFileNode = resourceFileNode.nextSiblingElement("resourceFile");
            }
            QDomElement installCommandFileNode = syntacticRulesNode.firstChildElement("installCommandFile");
            if(!installCommandFileNode.isNull() && !installCommandFileNode.attribute("pathSuffix").isNull())
                syntacticRulesInstallCommand = commandsRoot+"/"+installCommandFileNode.attribute("pathSuffix");
        }
        QDomElement idiomaticExpressionsNode = resourcesNode.firstChildElement("idiomaticExpressions");
        if(!idiomaticExpressionsNode.isNull())
        {
            QDomElement resourceFileNode = idiomaticExpressionsNode.firstChildElement("resourceFile");
            if(!resourceFileNode.isNull() && !resourceFileNode.attribute("pathSuffix").isNull())
                idiomaticExpressionsResourcePath = linguisticDataRoot+"/"+resourceFileNode.attribute("pathSuffix");
            QDomElement installCommandFileNode = idiomaticExpressionsNode.firstChildElement("installCommandFile");
            if(!installCommandFileNode.isNull() && !installCommandFileNode.attribute("pathSuffix").isNull())
                idiomaticExpressionsInstallCommand = commandsRoot+"/"+installCommandFileNode.attribute("pathSuffix");
        }
    }
    return true;
}

void ResourceTool::dictionnaryInit (QString resourcePath, QString installComand)
{
  qDebug() << "ResourceTool::dictionnaryInit" << resourcePath << installComand;
///@TODO Possible memory leak:  ensure that this object will be deleted
    dictionnaryModel = new ResourceEditorTableModel();
    dictionnaryModel->init(resourcePath, installComand);
    dictionnaryModel->columnCountPerEntry = DictionnaryEntry::columnCountPerEntry;
    dictionnaryModel->columnHeaders = DictionnaryEntry::columnHeaders;
    dictionnaryModel->loadData<DictionnaryEntry>();
    dictionnarySearchCb->addItem(DictionnaryEntry::allCategoryName);
    for(int i = 0; i < dictionnaryModel->categoryList.size(); i++)
        dictionnarySearchCb->addItem(dictionnaryModel->categoryList.at(i));
    dictionnaryView->init(dictionnaryModel);
    connect(dictionnaryModel, SIGNAL(dataChanged()), this, SLOT(dictionnarySearch()));
    connect(dictionnaryModel, SIGNAL(dataInstalled(bool)), this, SLOT(resourceInstalled(bool)));
    connect(dictionnarySearchBtn, SIGNAL(clicked(bool)), this, SLOT(dictionnarySearch()));
    connect(dictionnarySearchInp, SIGNAL(returnPressed()), this, SLOT(dictionnarySearch()));
    connect(dictionnaryView, SIGNAL(editEntryDlg(AbstractResourceEntry*)), this, SLOT(dictionnaryEditDlg(AbstractResourceEntry*)));
    connect(dictionnaryAddBtn, SIGNAL(clicked(bool)), dictionnaryView, SLOT(createEntry()));
    connect(dictionnaryInstallBtn, SIGNAL(clicked(bool)), this, SLOT(dictionnaryConfirmInstall()));
    dictionnarySearch();
}

void ResourceTool::dictionnaryEditDlg (AbstractResourceEntry* are)
{
///@TODO Possible memory leak:  ensure that this object will be deleted
    DictionnaryEntryEditDlg* dictionnaryEntryEditDlg = new DictionnaryEntryEditDlg();
    dictionnaryEntryEditDlg->init(dictionnaryModel, are);
}

void ResourceTool::dictionnarySearch ()
{
    QStringList searchArgs;
    searchArgs << dictionnarySearchInp->text();
    searchArgs << dictionnarySearchCb->currentText();
    int nbEntries = dictionnaryModel->searchEntries(searchArgs);
    dictionnaryCountLbl->setText(QString::number(nbEntries)+" entries found");
    dictionnaryView->resizeColumnsToContents();
}

void ResourceTool::dictionnaryConfirmInstall ()
{
    if(!checkEvaluateReady())
        return;
    dictionnaryView->confirmInstall();
}

void ResourceTool::nGramsInit (QStringList resourcePaths, QString installComand)
{
  qDebug() << "ResourceTool::nGramsInit" << resourcePaths << installComand;
///@TODO Possible memory leak:  ensure that this object will be deleted
    nGramsModel = new ResourceReaderTableModel();
    nGramsModel->init(resourcePaths, installComand);
    nGramsModel->columnCountPerEntry = NGramEntry::columnCountPerEntry;
    nGramsModel->columnHeaders = NGramEntry::columnHeaders;
    nGramsModel->loadData<NGramEntry>();
    NGramEntry::gramCategoryList.sort();
    nGramsInitCategoryCb(nGramsSearchCb1);
    nGramsInitCategoryCb(nGramsSearchCb2);
    nGramsInitCategoryCb(nGramsSearchCb3, true);
    nGramsView->init(nGramsModel);
    connect(nGramsModel, SIGNAL(dataChanged()), this, SLOT(nGramsSearch()));
    connect(nGramsModel, SIGNAL(dataInstalled(bool)), this, SLOT(nGramsInstalled(bool)));
    connect(nGramsResetBtn, SIGNAL(clicked(bool)), this, SLOT(nGramsSearchReset()));
    connect(nGramsShiftLeftBtn, SIGNAL(clicked(bool)), this, SLOT(nGramsShiftSearchLeft()));
    connect(nGramsShiftRightBtn, SIGNAL(clicked(bool)), this, SLOT(nGramsShiftSearchRight()));
    connect(nGramsSearchBtn, SIGNAL(clicked(bool)), this, SLOT(nGramsSearch()));
    connect(nGramsView, SIGNAL(clicked(const QModelIndex&)), this, SLOT(nGramsShowCorpus()));
    connect(nGramsEditCorpusBtn, SIGNAL(clicked(bool)), this, SLOT(nGramsEditCorpus()));
    connect(nGramsInstallBtn, SIGNAL(clicked(bool)), this, SLOT(nGramsConfirmInstall()));
    nGramsSearch();
}

void ResourceTool::nGramsInitCategoryCb (QComboBox* categoryCb, bool allowBigrams)
{
  categoryCb->addItem(NGramEntry::allCategoryName);
  for(int i = 0; i < NGramEntry::gramCategoryList.size(); i++)
      if(allowBigrams || NGramEntry::gramCategoryList.at(i) != NGramEntry::bigramsCategoryName)
        categoryCb->addItem(NGramEntry::gramCategoryList.at(i));
}

void ResourceTool::nGramsCorpusInit (QString symbolicCodesPath, QString corpusPath)
{
    qDebug() << "Loading NGrams symbolic codes: " << symbolicCodesPath;
    nGramsSymbolicCodesEdit->clear();
    ngramsSymbolicCodes.clear();
    QFile ngramsSymbolicCodesFile(symbolicCodesPath);
    if(!ngramsSymbolicCodesFile.exists() || !ngramsSymbolicCodesFile.open(QIODevice::ReadOnly | QIODevice::Text))
    {
        qDebug() << "Error opening NGrams symbolic codes file " << ngramsSymbolicCodesPath << " for reading";
//         return;
    }
    else
    {
      while(!ngramsSymbolicCodesFile.atEnd())
      {
          QString symbolicCode = QString::fromUtf8(ngramsSymbolicCodesFile.readLine()).trimmed();
          QStringList symbolicCodeParts = symbolicCode.split('\t');
          if(symbolicCodeParts.size() == 2)
          {
              nGramsSymbolicCodesEdit->appendPlainText(symbolicCode);
              ngramsSymbolicCodes[symbolicCodeParts[0]] = symbolicCodeParts[1];
          }
      }
      ngramsSymbolicCodesFile.close();
      nGramsSymbolicCodesEdit->moveCursor(QTextCursor::Start);
      nGramsSymbolicCodesEdit->ensureCursorVisible();
    }
    qDebug() << "Loading NGrams corpus path: " << corpusPath;
    ngramsMatches.clear();
    QFile ngramsCorpusFile(corpusPath);
    if(!ngramsCorpusFile.exists() || !ngramsCorpusFile.open(QIODevice::ReadOnly | QIODevice::Text))
    {
        qDebug() << "Error opening NGrams corpus file " << ngramsCorpusPath << " for reading";
        return;
    }
    QMap<QString, QStringList> curMatches;
    QMap<QString, QStringList>::iterator curMatchesIt, curMatchesItEnd;
    QString curText, prevText, prevPrevText, curCode, prevCode, prevPrevCode, curCat, prevCat, prevPrevCat;
    QString curSentence = "> ";
    while(!ngramsCorpusFile.atEnd())
    {
        QString currentWord = QString::fromUtf8(ngramsCorpusFile.readLine()).trimmed();
        QStringList currentWordParts = currentWord.split('\t');
        if(currentWordParts.size() == 2)
        {
            curCode = currentWordParts[1].trimmed();
            curCat = curCode;
            if(ngramsSymbolicCodes.contains(curCode))
                curCat = ngramsSymbolicCodes[curCode];
            curMatchesItEnd = curMatches.end();
            curText = currentWordParts[0].trimmed();
						curText += "<i>/" + curCat + "</i>";
//             qDebug() << "Entry" << curText << curCat;
            for(curMatchesIt = curMatches.begin(); curMatchesIt != curMatchesItEnd; curMatchesIt++)
                for(int i = 0; i < curMatchesIt.value().size(); i++)
                    curMatchesIt.value()[i] = curMatchesIt.value()[i] + curText + " ";
            if(!prevPrevCat.isEmpty())
            {
                QString triText = curSentence;
                triText += " <b>[ " +prevPrevText + " ";
                triText += prevText + " ";
								triText += curText + "]</b> ";
                curMatches[prevPrevCat + "/" + prevCat + "/" + curCat] << triText;
                curMatches[prevPrevCat + "/" + prevCat + "/" + NGramEntry::bigramsCategoryName] << triText;
                curSentence += prevPrevText + " ";
            }
            prevPrevText = prevText;
            prevPrevCode = prevCode;
						prevPrevCat = prevCat;
            prevText = curText;
            prevCode = curCode;
            prevCat = curCat;
            if(curCat == "L_PONCTU_FORTE" || curCat == "SENT")
            {
                curSentence = "<br/>> ";
                prevPrevText = prevText = "";
                for(curMatchesIt = curMatches.begin(); curMatchesIt != curMatchesItEnd; curMatchesIt++)
                    ngramsMatches[curMatchesIt.key()] << curMatchesIt.value();
                curMatches.clear();
            }
        }
    }
    ngramsCorpusFile.close();
}

void ResourceTool::nGramsShiftSearchLeft ()
{
  nGramsSearchCb1->setCurrentIndex(nGramsSearchCb2->currentIndex());
  nGramsSearchCb2->setCurrentIndex(nGramsSearchCb3->currentIndex());
  nGramsSearchCb3->setCurrentIndex(0);
  nGramsSearch();
}

void ResourceTool::nGramsShiftSearchRight ()
{
  nGramsSearchCb3->setCurrentIndex(nGramsSearchCb2->currentIndex());
  nGramsSearchCb2->setCurrentIndex(nGramsSearchCb1->currentIndex());
  nGramsSearchCb1->setCurrentIndex(0);
  nGramsSearch();
}

void ResourceTool::nGramsSearch ()
{<commit_msg>Adapt to Lima Libre  tagset<commit_after>/*
 *    Copyright 2002-2013 CEA LIST
 * 
 *    This file is part of LIMA.
 * 
 *    LIMA is free software: you can redistribute it and/or modify
 *    it under the terms of the GNU Affero General Public License as published by
 *    the Free Software Foundation, either version 3 of the License, or
 *    (at your option) any later version.
 * 
 *    LIMA is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 * 
 *    You should have received a copy of the GNU Affero General Public License
 *    along with LIMA.  If not, see <http://www.gnu.org/licenses/>
 */
#include "ResourceTool.h"
#include "ResourceEditorTableModel.h"
#include "ResourceFileEditDlg.h"
#include "DictionnaryEntry.h"
#include "DictionnaryEntryEditDlg.h"
#include "NGramEntry.h"
#include "SyntacticRuleEntry.h"
#include "IdiomaticExpressionEntry.h"
#include "IdiomaticExpressionEntryEditDlg.h"


#include <QtXml/QDomImplementation>
#include <QtXml/QDomDocument>
#include <QtCore/QtDebug>
#include <QtCore/QSettings>
#include <QtCore/QSharedMemory>
#include <QtCore/QTimer>
#include <QtCore/QFile>
#include <QtGui/QMainWindow>


using namespace Lima::Pelf;

ResourceTool::ResourceTool (QWidget* p):
    QMainWindow(p),
    Ui::PelfResourceToolWindow(),
    linguisticDataRoot(""),
    commandsRoot(""),
    evaluationStatus(EVALUATION_READY)
{
    setupUi(this);
}

bool ResourceTool::init ()
{
    qDebug() << "Starting application, loading settings...";
    settings = new QSettings("CEA LIST LIMA", "Pelf Resource Tool");
    restoreGeometry(settings->value("geometry").toByteArray());

    char* mmdist = getenv("LIMA_DIST");
    if (mmdist == 0) mmdist = "/usr";
    if(!xmlFileInit(QString(mmdist)+"/share/config/pelf/pelf-rt-conf.xml"))
    {
      qDebug() << "Unable to load configuration from file "<<(QString(mmdist)+"/share/config/pelf/pelf-rt-conf.xml")<<", aborting";
      return false;
    }

    initPelfSharedMemory();
    if(!dictionnaryResourcePath.isEmpty() && !dictionnaryInstallCommand.isEmpty())
        dictionnaryInit(dictionnaryResourcePath, dictionnaryInstallCommand);
    else
        editorTabsWidget->removeTab(editorTabsWidget->indexOf(dictionnaryTab));
    if(!ngramsResourcePaths.isEmpty() && !ngramsInstallCommand.isEmpty())
    {
        nGramsInit(ngramsResourcePaths, ngramsInstallCommand);
        if(!ngramsSymbolicCodesPath.isEmpty() && !ngramsCorpusPath.isEmpty())
            nGramsCorpusInit(ngramsSymbolicCodesPath, ngramsCorpusPath);
    }
    else
        editorTabsWidget->removeTab(editorTabsWidget->indexOf(nGramsTab));
    if(!syntacticRulesResourcePaths.isEmpty() && !syntacticRulesInstallCommand.isEmpty())
        syntacticRulesInit(syntacticRulesResourcePaths, syntacticRulesInstallCommand);
    else
        editorTabsWidget->removeTab(editorTabsWidget->indexOf(syntacticRulesTab));
    if(!idiomaticExpressionsResourcePath.isEmpty() && !idiomaticExpressionsInstallCommand.isEmpty())
        idiomaticExpressionsInit(idiomaticExpressionsResourcePath, idiomaticExpressionsInstallCommand);
    else
        editorTabsWidget->removeTab(editorTabsWidget->indexOf(idiomaticExpressionsTab));

    connect(evaluationBtn, SIGNAL(clicked(bool)), this, SLOT(confirmEvaluate()));

    messagesSplitter->restoreState(settings->value ("messagesSplitter").toByteArray());
    ngramsCategoriesSplitter->restoreState(settings->value ("ngramsCategoriesSplitter").toByteArray());
    ngramsOccurencesSplitter->restoreState(settings->value ("ngramsOccurencesSplitter").toByteArray());

    
    show();
    return true;
}

bool ResourceTool::xmlFileInit (QString xmlFilePath)
{
    QFile xmlFile(xmlFilePath);
    if(!xmlFile.open(QFile::ReadOnly)){
        qDebug() << "Unable to open xml configuration file, aborting";
        return false;
    }
    QDomDocument doc;
    QString errorMsg;
    if(!doc.setContent(&xmlFile, false, &errorMsg))
    {
        qDebug() << "Unable to parse xml file, aborting (" << errorMsg << ")";
        xmlFile.close();
        return false;
    }
    xmlFile.close();
    QDomElement configurationNode = doc.documentElement();
    if(configurationNode.tagName() != "configuration" )
    {
        qDebug() << "Unrecognized xml file format, aborting";
        return false;
    }

    QDomElement linguisticDataRootNode = configurationNode.firstChildElement("linguisticDataRoot");
    if(!linguisticDataRootNode.isNull() && !linguisticDataRootNode.attribute("path").isNull())
        linguisticDataRoot = linguisticDataRootNode.attribute("path");
    if(linguisticDataRoot.isEmpty())
        linguisticDataRoot = getenv("LINGUISTIC_DATA_ROOT");
    if(linguisticDataRoot.isEmpty())
    {
      qDebug() << "Linguistic data root is not defined";
      return false;
    }
    setenv("LINGUISTIC_DATA_ROOT", linguisticDataRoot.toUtf8(), 1);
    qDebug() << "Retrieved and set env variable linguistic data root to " << linguisticDataRoot;

    QDomElement commandsRootNode = configurationNode.firstChildElement("commandsRoot");
    if(!commandsRootNode.isNull() && !commandsRootNode.attribute("path").isNull())
        commandsRoot = commandsRootNode.attribute("path");
    if(commandsRoot.isEmpty())
    {
      commandsRoot = getenv("LIMA_DIST");
      if (commandsRoot.isEmpty())
      {
        commandsRoot = QString("/usr");
      }
    }
    commandsRoot += "/share/apps/pelf/commands";
    setenv("PELF_COMMANDS_ROOT", commandsRoot.toUtf8(), 1);
    qDebug() << "Retrieved command root " << commandsRoot;

    QDomElement resourcesNode = configurationNode.firstChildElement("resources");
    if(!resourcesNode.isNull())
    {
        QDomElement dictionnaryNode = resourcesNode.firstChildElement("dictionnary");
        if(!dictionnaryNode.isNull())
        {
            QDomElement resourceFileNode = dictionnaryNode.firstChildElement("resourceFile");
            if(!resourceFileNode.isNull() && !resourceFileNode.attribute("pathSuffix").isNull())
            {
                dictionnaryResourcePath = resourceFileNode.attribute("pathSuffix");
                if (!dictionnaryResourcePath.startsWith('/'))
                  dictionnaryResourcePath.prepend(linguisticDataRoot+"/");
            }
            QDomElement installCommandFileNode = dictionnaryNode.firstChildElement("installCommandFile");
            if(!installCommandFileNode.isNull() && !installCommandFileNode.attribute("pathSuffix").isNull())
                dictionnaryInstallCommand = commandsRoot+"/"+installCommandFileNode.attribute("pathSuffix");
        }
        QDomElement ngramsNode = resourcesNode.firstChildElement("ngrams");
        if(!ngramsNode.isNull())
        {
            QDomElement resourceFileNode = ngramsNode.firstChildElement("resourceFile");
            while(!resourceFileNode.isNull())
            {
                if(!resourceFileNode.isNull() && !resourceFileNode.attribute("pathSuffix").isNull())
								{
									if (resourceFileNode.attribute("pathSuffix").startsWith('/'))
                    ngramsResourcePaths << resourceFileNode.attribute("pathSuffix");
									else
										ngramsResourcePaths << linguisticDataRoot+"/"+resourceFileNode.attribute("pathSuffix");
									resourceFileNode = resourceFileNode.nextSiblingElement("resourceFile");
								}
            }
            QDomElement installCommandFileNode = ngramsNode.firstChildElement("installCommandFile");
            if(!installCommandFileNode.isNull() && !installCommandFileNode.attribute("pathSuffix").isNull())
                ngramsInstallCommand = commandsRoot+"/"+installCommandFileNode.attribute("pathSuffix");
            QDomElement symbolicCodeFileNode = ngramsNode.firstChildElement("symbolicCodeFile");
            if(!symbolicCodeFileNode.isNull() && !symbolicCodeFileNode.attribute("pathSuffix").isNull())
                ngramsSymbolicCodesPath = linguisticDataRoot+"/"+symbolicCodeFileNode.attribute("pathSuffix");
            QDomElement corpusFileNode = ngramsNode.firstChildElement("corpusFile");
            if(!corpusFileNode.isNull() && !corpusFileNode.attribute("pathSuffix").isNull())
            {
              ngramsCorpusPath = corpusFileNode.attribute("pathSuffix");
              if (!ngramsCorpusPath.startsWith('/'))
                ngramsCorpusPath.prepend(linguisticDataRoot+"/");
            }
        }
        QDomElement syntacticRulesNode = resourcesNode.firstChildElement("syntacticRules");
        if(!syntacticRulesNode.isNull())
        {
            QDomElement resourceFileNode = syntacticRulesNode.firstChildElement("resourceFile");
            while(!resourceFileNode.isNull())
            {
                if(!resourceFileNode.isNull() && !resourceFileNode.attribute("pathSuffix").isNull())
                    syntacticRulesResourcePaths << linguisticDataRoot+"/"+resourceFileNode.attribute("pathSuffix");
                resourceFileNode = resourceFileNode.nextSiblingElement("resourceFile");
            }
            QDomElement installCommandFileNode = syntacticRulesNode.firstChildElement("installCommandFile");
            if(!installCommandFileNode.isNull() && !installCommandFileNode.attribute("pathSuffix").isNull())
                syntacticRulesInstallCommand = commandsRoot+"/"+installCommandFileNode.attribute("pathSuffix");
        }
        QDomElement idiomaticExpressionsNode = resourcesNode.firstChildElement("idiomaticExpressions");
        if(!idiomaticExpressionsNode.isNull())
        {
            QDomElement resourceFileNode = idiomaticExpressionsNode.firstChildElement("resourceFile");
            if(!resourceFileNode.isNull() && !resourceFileNode.attribute("pathSuffix").isNull())
                idiomaticExpressionsResourcePath = linguisticDataRoot+"/"+resourceFileNode.attribute("pathSuffix");
            QDomElement installCommandFileNode = idiomaticExpressionsNode.firstChildElement("installCommandFile");
            if(!installCommandFileNode.isNull() && !installCommandFileNode.attribute("pathSuffix").isNull())
                idiomaticExpressionsInstallCommand = commandsRoot+"/"+installCommandFileNode.attribute("pathSuffix");
        }
    }
    return true;
}

void ResourceTool::dictionnaryInit (QString resourcePath, QString installComand)
{
  qDebug() << "ResourceTool::dictionnaryInit" << resourcePath << installComand;
///@TODO Possible memory leak:  ensure that this object will be deleted
    dictionnaryModel = new ResourceEditorTableModel();
    dictionnaryModel->init(resourcePath, installComand);
    dictionnaryModel->columnCountPerEntry = DictionnaryEntry::columnCountPerEntry;
    dictionnaryModel->columnHeaders = DictionnaryEntry::columnHeaders;
    dictionnaryModel->loadData<DictionnaryEntry>();
    dictionnarySearchCb->addItem(DictionnaryEntry::allCategoryName);
    for(int i = 0; i < dictionnaryModel->categoryList.size(); i++)
        dictionnarySearchCb->addItem(dictionnaryModel->categoryList.at(i));
    dictionnaryView->init(dictionnaryModel);
    connect(dictionnaryModel, SIGNAL(dataChanged()), this, SLOT(dictionnarySearch()));
    connect(dictionnaryModel, SIGNAL(dataInstalled(bool)), this, SLOT(resourceInstalled(bool)));
    connect(dictionnarySearchBtn, SIGNAL(clicked(bool)), this, SLOT(dictionnarySearch()));
    connect(dictionnarySearchInp, SIGNAL(returnPressed()), this, SLOT(dictionnarySearch()));
    connect(dictionnaryView, SIGNAL(editEntryDlg(AbstractResourceEntry*)), this, SLOT(dictionnaryEditDlg(AbstractResourceEntry*)));
    connect(dictionnaryAddBtn, SIGNAL(clicked(bool)), dictionnaryView, SLOT(createEntry()));
    connect(dictionnaryInstallBtn, SIGNAL(clicked(bool)), this, SLOT(dictionnaryConfirmInstall()));
    dictionnarySearch();
}

void ResourceTool::dictionnaryEditDlg (AbstractResourceEntry* are)
{
///@TODO Possible memory leak:  ensure that this object will be deleted
    DictionnaryEntryEditDlg* dictionnaryEntryEditDlg = new DictionnaryEntryEditDlg();
    dictionnaryEntryEditDlg->init(dictionnaryModel, are);
}

void ResourceTool::dictionnarySearch ()
{
    QStringList searchArgs;
    searchArgs << dictionnarySearchInp->text();
    searchArgs << dictionnarySearchCb->currentText();
    int nbEntries = dictionnaryModel->searchEntries(searchArgs);
    dictionnaryCountLbl->setText(QString::number(nbEntries)+" entries found");
    dictionnaryView->resizeColumnsToContents();
}

void ResourceTool::dictionnaryConfirmInstall ()
{
    if(!checkEvaluateReady())
        return;
    dictionnaryView->confirmInstall();
}

void ResourceTool::nGramsInit (QStringList resourcePaths, QString installComand)
{
  qDebug() << "ResourceTool::nGramsInit" << resourcePaths << installComand;
///@TODO Possible memory leak:  ensure that this object will be deleted
    nGramsModel = new ResourceReaderTableModel();
    nGramsModel->init(resourcePaths, installComand);
    nGramsModel->columnCountPerEntry = NGramEntry::columnCountPerEntry;
    nGramsModel->columnHeaders = NGramEntry::columnHeaders;
    nGramsModel->loadData<NGramEntry>();
    NGramEntry::gramCategoryList.sort();
    nGramsInitCategoryCb(nGramsSearchCb1);
    nGramsInitCategoryCb(nGramsSearchCb2);
    nGramsInitCategoryCb(nGramsSearchCb3, true);
    nGramsView->init(nGramsModel);
    connect(nGramsModel, SIGNAL(dataChanged()), this, SLOT(nGramsSearch()));
    connect(nGramsModel, SIGNAL(dataInstalled(bool)), this, SLOT(nGramsInstalled(bool)));
    connect(nGramsResetBtn, SIGNAL(clicked(bool)), this, SLOT(nGramsSearchReset()));
    connect(nGramsShiftLeftBtn, SIGNAL(clicked(bool)), this, SLOT(nGramsShiftSearchLeft()));
    connect(nGramsShiftRightBtn, SIGNAL(clicked(bool)), this, SLOT(nGramsShiftSearchRight()));
    connect(nGramsSearchBtn, SIGNAL(clicked(bool)), this, SLOT(nGramsSearch()));
    connect(nGramsView, SIGNAL(clicked(const QModelIndex&)), this, SLOT(nGramsShowCorpus()));
    connect(nGramsEditCorpusBtn, SIGNAL(clicked(bool)), this, SLOT(nGramsEditCorpus()));
    connect(nGramsInstallBtn, SIGNAL(clicked(bool)), this, SLOT(nGramsConfirmInstall()));
    nGramsSearch();
}

void ResourceTool::nGramsInitCategoryCb (QComboBox* categoryCb, bool allowBigrams)
{
  categoryCb->addItem(NGramEntry::allCategoryName);
  for(int i = 0; i < NGramEntry::gramCategoryList.size(); i++)
      if(allowBigrams || NGramEntry::gramCategoryList.at(i) != NGramEntry::bigramsCategoryName)
        categoryCb->addItem(NGramEntry::gramCategoryList.at(i));
}

void ResourceTool::nGramsCorpusInit (QString symbolicCodesPath, QString corpusPath)
{
    qDebug() << "Loading NGrams symbolic codes: " << symbolicCodesPath;
    nGramsSymbolicCodesEdit->clear();
    ngramsSymbolicCodes.clear();
    QFile ngramsSymbolicCodesFile(symbolicCodesPath);
    if(!ngramsSymbolicCodesFile.exists() || !ngramsSymbolicCodesFile.open(QIODevice::ReadOnly | QIODevice::Text))
    {
        qDebug() << "Error opening NGrams symbolic codes file " << ngramsSymbolicCodesPath << " for reading";
//         return;
    }
    else
    {
      while(!ngramsSymbolicCodesFile.atEnd())
      {
          QString symbolicCode = QString::fromUtf8(ngramsSymbolicCodesFile.readLine()).trimmed();
          QStringList symbolicCodeParts = symbolicCode.split('\t');
          if(symbolicCodeParts.size() == 2)
          {
              nGramsSymbolicCodesEdit->appendPlainText(symbolicCode);
              ngramsSymbolicCodes[symbolicCodeParts[0]] = symbolicCodeParts[1];
          }
      }
      ngramsSymbolicCodesFile.close();
      nGramsSymbolicCodesEdit->moveCursor(QTextCursor::Start);
      nGramsSymbolicCodesEdit->ensureCursorVisible();
    }
    qDebug() << "Loading NGrams corpus path: " << corpusPath;
    ngramsMatches.clear();
    QFile ngramsCorpusFile(corpusPath);
    if(!ngramsCorpusFile.exists() || !ngramsCorpusFile.open(QIODevice::ReadOnly | QIODevice::Text))
    {
        qDebug() << "Error opening NGrams corpus file " << ngramsCorpusPath << " for reading";
        return;
    }
    QMap<QString, QStringList> curMatches;
    QMap<QString, QStringList>::iterator curMatchesIt, curMatchesItEnd;
    QString curText, prevText, prevPrevText, curCode, prevCode, prevPrevCode, curCat, prevCat, prevPrevCat;
    QString curSentence = "> ";
    while(!ngramsCorpusFile.atEnd())
    {
        QString currentWord = QString::fromUtf8(ngramsCorpusFile.readLine()).trimmed();
        QStringList currentWordParts = currentWord.split('\t');
        if(currentWordParts.size() == 2)
        {
            curCode = currentWordParts[1].trimmed();
            curCat = curCode;
            if(ngramsSymbolicCodes.contains(curCode))
                curCat = ngramsSymbolicCodes[curCode];
            curMatchesItEnd = curMatches.end();
            curText = currentWordParts[0].trimmed();
						curText += "<i>/" + curCat + "</i>";
//             qDebug() << "Entry" << curText << curCat;
            for(curMatchesIt = curMatches.begin(); curMatchesIt != curMatchesItEnd; curMatchesIt++)
                for(int i = 0; i < curMatchesIt.value().size(); i++)
                    curMatchesIt.value()[i] = curMatchesIt.value()[i] + curText + " ";
            if(!prevPrevCat.isEmpty())
            {
                QString triText = curSentence;
                triText += " <b>[ " +prevPrevText + " ";
                triText += prevText + " ";
								triText += curText + "]</b> ";
                curMatches[prevPrevCat + "/" + prevCat + "/" + curCat] << triText;
                curMatches[prevPrevCat + "/" + prevCat + "/" + NGramEntry::bigramsCategoryName] << triText;
                curSentence += prevPrevText + " ";
            }
            prevPrevText = prevText;
            prevPrevCode = prevCode;
						prevPrevCat = prevCat;
            prevText = curText;
            prevCode = curCode;
            prevCat = curCat;
            if(curCat == "PONCTU_FORTE" || curCat == "SENT")
            {
                curSentence = "<br/>> ";
                prevPrevText = prevText = "";
                for(curMatchesIt = curMatches.begin(); curMatchesIt != curMatchesItEnd; curMatchesIt++)
                    ngramsMatches[curMatchesIt.key()] << curMatchesIt.value();
                curMatches.clear();
            }
        }
    }
    ngramsCorpusFile.close();
}

void ResourceTool::nGramsShiftSearchLeft ()
{
  nGramsSearchCb1->setCurrentIndex(nGramsSearchCb2->currentIndex());
  nGramsSearchCb2->setCurrentIndex(nGramsSearchCb3->currentIndex());
  nGramsSearchCb3->setCurrentIndex(0);
  nGramsSearch();
}

void ResourceTool::nGramsShiftSearchRight ()
{
  nGramsSearchCb3->setCurrentIndex(nGramsSearchCb2->currentIndex());
  nGramsSearchCb2->setCurrentIndex(nGramsSearchCb1->currentIndex());
  nGramsSearchCb1->setCurrentIndex(0);
  nGramsSearch();
}

void ResourceTool::nGramsSearch ()
{
{'fraction_non_alphanumeric': 0.12174558566201814, 'fraction_numerical': 0.0021210032345299326, 'mean_word_length': 3.974808757583751, 'pattern_counts': {'":': 0, '<': 209, '<?xml version=': 0, '>': 147, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 28}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4725664', 'n_tokens_mistral': 11430, 'n_tokens_neox': 11009, 'n_words': 1696}