<commit_before>/*
    Copyright 2002-2015 CEA LIST

    This file is part of LIMA.

    LIMA is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    LIMA is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with LIMA.  If not, see <http://www.gnu.org/licenses/>
*/
#include "EnhancedAnalysisDictionary.h"
#include "EnhancedAnalysisDictionaryIterator.h"
#include "AbstractAccessResource.h"
#include "common/XMLConfigurationFiles/xmlConfigurationFileExceptions.h"
#include "common/AbstractFactoryPattern/SimpleFactory.h"
#include "common/MediaticData/mediaticData.h"
#include "linguisticProcessing/core/LinguisticResources/LinguisticResources.h"

#include <iostream>
#include <QFileSystemWatcher>
#include <QReadWriteLock>
#include <QWriteLocker>
#include <QReadLocker>

using namespace Lima::Common;
using namespace Lima::Common::XMLConfigurationFiles;
using namespace Lima::Common::MediaticData;
using namespace std;

namespace Lima
{

namespace LinguisticProcessing
{

namespace AnalysisDict
{

SimpleFactory<AbstractResource,EnhancedAnalysisDictionary> enhancedAnalysisDictionaryFactory(ENHANCEDANALYSISDICTIONARY_CLASSID);


class EnhancedAnalysisDictionaryPrivate
{
  friend class EnhancedAnalysisDictionary;
  
  EnhancedAnalysisDictionaryPrivate();
  
  EnhancedAnalysisDictionaryPrivate(
    FsaStringsPool* sp,
    Lima::Common::AbstractAccessByString* access,
    const std::string& dataFile);

  ~EnhancedAnalysisDictionaryPrivate();

  Lima::Common::AbstractAccessByString* m_access;
  DictionaryData* m_dicoData;
  Lima::FsaStringsPool* m_sp;
  bool m_isMainKeys;
  // A read/write locker to prevent accessing m_dicoData during its loading
  QReadWriteLock m_lock;
};

EnhancedAnalysisDictionaryPrivate::EnhancedAnalysisDictionaryPrivate() :
    m_access(0),
    m_dicoData(new DictionaryData()),
    m_sp(0)
{
}

EnhancedAnalysisDictionaryPrivate::EnhancedAnalysisDictionaryPrivate(
      FsaStringsPool* sp,
      Lima::Common::AbstractAccessByString* access,
      const std::string& dataFile) :
   m_access(access),
   m_dicoData(new DictionaryData()),
   m_sp(sp)
{
  QWriteLocker locker(&m_lock);
  m_dicoData->loadBinaryFile(dataFile);
}


EnhancedAnalysisDictionaryPrivate::~EnhancedAnalysisDictionaryPrivate()
{
}


EnhancedAnalysisDictionary::EnhancedAnalysisDictionary(const QString& dataFilePath)
    : AbstractAnalysisDictionary(),
    m_d(new EnhancedAnalysisDictionaryPrivate())
{
  ANALYSISDICTLOGINIT;
//   LDEBUG  << "EnhancedAnalysisDictionary::EnhancedAnalysisDictionary";
  connect(this,SIGNAL(resourceFileChanged(QString)),this,SLOT(dictionaryFileChanged(QString)));
  
  if (!dataFilePath.isEmpty())
  {
    resourceFileWatcher().addPath(dataFilePath);
    QWriteLocker locker(&m_d->m_lock);
    m_d->m_dicoData->loadBinaryFile(dataFilePath.toUtf8().constData());
  }
}

EnhancedAnalysisDictionary::EnhancedAnalysisDictionary(
      FsaStringsPool* sp,
      Lima::Common::AbstractAccessByString* access,
      const std::string& dataFile) :
   m_d(new EnhancedAnalysisDictionaryPrivate(sp, access, dataFile))
{
  ANALYSISDICTLOGINIT;
//   LDEBUG  << "EnhancedAnalysisDictionary::EnhancedAnalysisDictionary" << dataFile.c_str();
  connect(this,SIGNAL(resourceFileChanged(QString)),this,SLOT(dictionaryFileChanged(QString)));
  resourceFileWatcher().addPath(QString::fromUtf8(dataFile.c_str()));
  QWriteLocker locker(&m_d->m_lock);
  m_d->m_dicoData->loadBinaryFile(dataFile);
}


EnhancedAnalysisDictionary::~EnhancedAnalysisDictionary()
{
  delete m_d;
}

void EnhancedAnalysisDictionary::init(
  Common::XMLConfigurationFiles::GroupConfigurationStructure& unitConfiguration,
  Manager* manager)
{
  ANALYSISDICTLOGINIT;
//   LDEBUG  << "EnhancedAnalysisDictionary::init";
  MediaId language=manager->getInitializationParameters().language;
  m_d->m_sp=&Common::MediaticData::MediaticData::changeable().stringsPool(language);
  try
  {
    string accessId=unitConfiguration.getParamsValueAtKey("accessKeys");
    const AbstractResource* res=LinguisticResources::single().getResource(language,accessId);
    const AbstractAccessResource* aar=static_cast<const AbstractAccessResource*>(res);
    m_d->m_isMainKeys=aar->isMainKeys();
    m_d->m_access=aar->getAccessByString();
  }
  catch (NoSuchParam& )
  {
    LERROR << "no param 'accessKeys' in EnhancedAnalysisDictionary group for language " << (int) language;
    throw InvalidConfiguration();
  }
  try
  {
    std::string binaryFilePath = Common::MediaticData::MediaticData::single().getResourcesPath() + "/" + unitConfiguration.getParamsValueAtKey("dictionaryValuesFile");
    resourceFileWatcher().addPath(QString::fromUtf8(binaryFilePath.c_str()));
    QWriteLocker locker(&m_d->m_lock);
    m_d->m_dicoData->loadBinaryFile(binaryFilePath);
  }
  catch (NoSuchList& )
  {
    LERROR << "no list 'dictionaryValues' in EnhancedAnalysisDictionary group for language " << (int) language;
    throw InvalidConfiguration();
  }


}

void EnhancedAnalysisDictionary::dictionaryFileChanged ( const QString & path )
{
  ANALYSISDICTLOGINIT;
  LINFO << "EnhancedAnalysisDictionary::dictionaryFileChanged" << path;
  QWriteLocker locker(&m_d->m_lock);
  if (m_d->m_dicoData != 0)
    delete m_d->m_dicoData;
  m_d->m_dicoData = new DictionaryData();
  m_d->m_dicoData->loadBinaryFile(path.toUtf8().constData());
}

uint64_t EnhancedAnalysisDictionary::getSize() const
{
  QReadLocker locker(&m_d->m_lock);
  return m_d->m_dicoData->getSize();
}

DictionaryEntry EnhancedAnalysisDictionary::getEntry(const StringsPoolIndex wordId, const Lima::LimaString& word) const
{
  if (m_d->m_isMainKeys) return getEntryData(wordId);
  return getEntryData(static_cast<StringsPoolIndex>(m_d->m_access->getIndex(word)));
}

DictionaryEntry EnhancedAnalysisDictionary::getEntry(const Lima::LimaString& word) const
{
  return getEntryData( static_cast<StringsPoolIndex>(m_d->m_access->getIndex(word)) );
}

DictionaryEntry EnhancedAnalysisDictionary::getEntry(const StringsPoolIndex wordId) const
{
  if (m_d->m_isMainKeys) return getEntryData(wordId);
  return getEntryData( static_cast<StringsPoolIndex>(m_d->m_access->getIndex((*m_d->m_sp)[wordId])) );
}


DictionaryEntry EnhancedAnalysisDictionary::getEntryData(const StringsPoolIndex wordId) const
{<commit_msg>Do not try to reload a deleted resource file<commit_after>/*
    Copyright 2002-2015 CEA LIST

    This file is part of LIMA.

    LIMA is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    LIMA is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with LIMA.  If not, see <http://www.gnu.org/licenses/>
*/
#include "EnhancedAnalysisDictionary.h"
#include "EnhancedAnalysisDictionaryIterator.h"
#include "AbstractAccessResource.h"
#include "common/XMLConfigurationFiles/xmlConfigurationFileExceptions.h"
#include "common/AbstractFactoryPattern/SimpleFactory.h"
#include "common/MediaticData/mediaticData.h"
#include "linguisticProcessing/core/LinguisticResources/LinguisticResources.h"

#include <iostream>
#include <QReadWriteLock>
#include <QWriteLocker>
#include <QReadLocker>
#include <QFileInfo>

using namespace Lima;
using namespace Lima::Common;
using namespace Lima::Common::XMLConfigurationFiles;
using namespace Lima::Common::MediaticData;
using namespace std;

namespace Lima
{

namespace LinguisticProcessing
{

namespace AnalysisDict
{

SimpleFactory<AbstractResource,EnhancedAnalysisDictionary> enhancedAnalysisDictionaryFactory(ENHANCEDANALYSISDICTIONARY_CLASSID);


class EnhancedAnalysisDictionaryPrivate
{
  friend class EnhancedAnalysisDictionary;
  
  EnhancedAnalysisDictionaryPrivate();
  
  EnhancedAnalysisDictionaryPrivate(
    FsaStringsPool* sp,
    Lima::Common::AbstractAccessByString* access,
    const std::string& dataFile);

  ~EnhancedAnalysisDictionaryPrivate();

  Lima::Common::AbstractAccessByString* m_access;
  DictionaryData* m_dicoData;
  Lima::FsaStringsPool* m_sp;
  bool m_isMainKeys;
  // A read/write locker to prevent accessing m_dicoData during its loading
  QReadWriteLock m_lock;
};

EnhancedAnalysisDictionaryPrivate::EnhancedAnalysisDictionaryPrivate() :
    m_access(0),
    m_dicoData(new DictionaryData()),
    m_sp(0)
{
}

EnhancedAnalysisDictionaryPrivate::EnhancedAnalysisDictionaryPrivate(
      FsaStringsPool* sp,
      Lima::Common::AbstractAccessByString* access,
      const std::string& dataFile) :
   m_access(access),
   m_dicoData(new DictionaryData()),
   m_sp(sp)
{
  QWriteLocker locker(&m_lock);
  m_dicoData->loadBinaryFile(dataFile);
}


EnhancedAnalysisDictionaryPrivate::~EnhancedAnalysisDictionaryPrivate()
{
}


EnhancedAnalysisDictionary::EnhancedAnalysisDictionary(const QString& dataFilePath)
    : AbstractAnalysisDictionary(),
    m_d(new EnhancedAnalysisDictionaryPrivate())
{
  ANALYSISDICTLOGINIT;
//   LDEBUG  << "EnhancedAnalysisDictionary::EnhancedAnalysisDictionary";
  connect(this,SIGNAL(resourceFileChanged(QString)),this,SLOT(dictionaryFileChanged(QString)));
  
  if (!dataFilePath.isEmpty())
  {
    resourceFileWatcher().addPath(dataFilePath);
    QWriteLocker locker(&m_d->m_lock);
    m_d->m_dicoData->loadBinaryFile(dataFilePath.toUtf8().constData());
  }
}

EnhancedAnalysisDictionary::EnhancedAnalysisDictionary(
      FsaStringsPool* sp,
      Lima::Common::AbstractAccessByString* access,
      const std::string& dataFile) :
   m_d(new EnhancedAnalysisDictionaryPrivate(sp, access, dataFile))
{
  ANALYSISDICTLOGINIT;
//   LDEBUG  << "EnhancedAnalysisDictionary::EnhancedAnalysisDictionary" << dataFile.c_str();
  connect(this,SIGNAL(resourceFileChanged(QString)),this,SLOT(dictionaryFileChanged(QString)));
  resourceFileWatcher().addPath(QString::fromUtf8(dataFile.c_str()));
  QWriteLocker locker(&m_d->m_lock);
  m_d->m_dicoData->loadBinaryFile(dataFile);
}


EnhancedAnalysisDictionary::~EnhancedAnalysisDictionary()
{
  delete m_d;
}

void EnhancedAnalysisDictionary::init(
  Common::XMLConfigurationFiles::GroupConfigurationStructure& unitConfiguration,
  Manager* manager)
{
  ANALYSISDICTLOGINIT;
//   LDEBUG  << "EnhancedAnalysisDictionary::init";
  MediaId language=manager->getInitializationParameters().language;
  m_d->m_sp=&Common::MediaticData::MediaticData::changeable().stringsPool(language);
  try
  {
    string accessId=unitConfiguration.getParamsValueAtKey("accessKeys");
    const AbstractResource* res=LinguisticResources::single().getResource(language,accessId);
    const AbstractAccessResource* aar=static_cast<const AbstractAccessResource*>(res);
    m_d->m_isMainKeys=aar->isMainKeys();
    m_d->m_access=aar->getAccessByString();
  }
  catch (NoSuchParam& )
  {
    LERROR << "no param 'accessKeys' in EnhancedAnalysisDictionary group for language " << (int) language;
    throw InvalidConfiguration();
  }
  try
  {
    std::string binaryFilePath = Common::MediaticData::MediaticData::single().getResourcesPath() + "/" + unitConfiguration.getParamsValueAtKey("dictionaryValuesFile");
    resourceFileWatcher().addPath(QString::fromUtf8(binaryFilePath.c_str()));
    QWriteLocker locker(&m_d->m_lock);
    m_d->m_dicoData->loadBinaryFile(binaryFilePath);
  }
  catch (NoSuchList& )
  {
    LERROR << "no list 'dictionaryValues' in EnhancedAnalysisDictionary group for language " << (int) language;
    throw InvalidConfiguration();
  }


}

void EnhancedAnalysisDictionary::dictionaryFileChanged ( const QString & path )
{
  ANALYSISDICTLOGINIT;
  // Check if the file exists as, when a file is replaced, dictionaryFileChanged can be triggered 
  // two times, when it is first suppressed and when the new version is available. One should not 
  // try to load the missing file
  if (QFileInfo(path).exists())
  {
    LINFO << "EnhancedAnalysisDictionary::dictionaryFileChanged reload" << path;
    QWriteLocker locker(&m_d->m_lock);
    if (m_d->m_dicoData != 0)
      delete m_d->m_dicoData;
    m_d->m_dicoData = new DictionaryData();
    m_d->m_dicoData->loadBinaryFile(path.toUtf8().constData());
  }
  else
  {
    LINFO << "EnhancedAnalysisDictionary::dictionaryFileChanged deleted, ignoring" << path;
  }
}

uint64_t EnhancedAnalysisDictionary::getSize() const
{
  QReadLocker locker(&m_d->m_lock);
  return m_d->m_dicoData->getSize();
}

DictionaryEntry EnhancedAnalysisDictionary::getEntry(const StringsPoolIndex wordId, const Lima::LimaString& word) const
{
  if (m_d->m_isMainKeys) return getEntryData(wordId);
  return getEntryData(static_cast<StringsPoolIndex>(m_d->m_access->getIndex(word)));
}

DictionaryEntry EnhancedAnalysisDictionary::getEntry(const Lima::LimaString& word) const
{
  return getEntryData( static_cast<StringsPoolIndex>(m_d->m_access->getIndex(word)) );
}

DictionaryEntry EnhancedAnalysisDictionary::getEntry(const StringsPoolIndex wordId) const
{
  if (m_d->m_isMainKeys) return getEntryData(wordId);
  return getEntryData( static_cast<StringsPoolIndex>(m_d->m_access->getIndex((*m_d->m_sp)[wordId])) );
}


DictionaryEntry EnhancedAnalysisDictionary::getEntryData(const StringsPoolIndex wordId) const
{
{'fraction_non_alphanumeric': 0.10466291323805399, 'fraction_numerical': 0.00259852750108272, 'mean_word_length': 6.2539267015706805, 'pattern_counts': {'":': 0, '<': 69, '<?xml version=': 0, '>': 93, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6005122', 'n_tokens_mistral': 4310, 'n_tokens_neox': 4205, 'n_words': 816}