<commit_before>//
// <NAME> <<EMAIL>>
//

#include "LM/Base.h"
#include "LM/LDHT.h"
#include "../FFState.h"
#include "../TypeDef.h"
#include "../Hypothesis.h"

#include <LDHT/Client.h>
#include <LDHT/ClientLocal.h>
#include <LDHT/NewNgram.h>
#include <LDHT/FactoryCollection.h>

#include <boost/thread/tss.hpp>

namespace Moses {

struct LDHTLMState : public FFState {
  LDHT::NewNgram gram_fingerprints;

  int Compare(const FFState& uncast_other) const {
    const LDHTLMState &other = static_cast<const LDHTLMState&>(uncast_other);
    return gram_fingerprints.compareMoses(other.gram_fingerprints);
  }

  void copyFrom(const LDHTLMState& other) {
    gram_fingerprints.copyFrom(other.gram_fingerprints);
  }
};

class LanguageModelLDHT : public LanguageModel {
public:
    LanguageModelLDHT();
    LanguageModelLDHT(const std::string& path,
                      ScoreIndexManager& manager,
                      FactorType factorType);
    LanguageModelLDHT(ScoreIndexManager& manager,
                      LanguageModelLDHT& copyFrom);
    std::string GetScoreProducerDescription(unsigned) const {
      std::ostringstream oss;
      oss << "LM_" << LDHT::NewNgram::k_max_order << "gram";
      return oss.str();
    }
    LDHT::Client* getClientUnsafe() const;
    LDHT::Client* getClientSafe();
    LDHT::Client* initTSSClient();
    virtual ~LanguageModelLDHT();
    virtual LanguageModel* Duplicate(
            ScoreIndexManager& scoreIndexManager) const;
    virtual void InitializeBeforeSentenceProcessing();
    virtual void CleanUpAfterSentenceProcessing();
    virtual const FFState* EmptyHypothesisState(const InputType& input) const;
    virtual bool Useable(const Phrase& phrase) const;
    virtual void CalcScore(const Phrase& phrase,
                           float& fullScore,
                           float& ngramScore,
                           std::size_t& oovCount) const;
    FFState* Evaluate(const Hypothesis& hypo,
                      const FFState* input_state,
                      ScoreComponentCollection* score_output) const;
    FFState* EvaluateChart(const ChartHypothesis& hypo,
                           int featureID,
                           ScoreComponentCollection* accumulator) const;

protected:
    boost::thread_specific_ptr<LDHT::Client> m_client;
    std::string m_configPath;
    FactorType m_factorType;

};

LanguageModel* ConstructLDHTLM(const std::string& path,
                               ScoreIndexManager& manager,
                               FactorType factorType) {
    return new LanguageModelLDHT(path, manager, factorType);
}

LanguageModelLDHT::LanguageModelLDHT() : LanguageModel(), m_client(NULL) {
    m_enableOOVFeature = false;
}

LanguageModelLDHT::LanguageModelLDHT(ScoreIndexManager& manager,
                                     LanguageModelLDHT& copyFrom) {
    //m_client = copyFrom.m_client;
    m_factorType = copyFrom.m_factorType;
    m_configPath = copyFrom.m_configPath;
    Init(manager);
}

LanguageModelLDHT::LanguageModelLDHT(const std::string& path,
                                     ScoreIndexManager& manager,
                                     FactorType factorType)
                                            : m_factorType(factorType) {
    m_configPath = path;
    Init(manager);
}

LanguageModelLDHT::~LanguageModelLDHT() {
    // TODO(wilson): should cleanup for each individual thread.
    delete getClientSafe();
}

LanguageModel* LanguageModelLDHT::Duplicate(
            ScoreIndexManager& scoreIndexManager) const {
    return NULL;
}

// Check that there is a TSS Client instance, and instantiate one if
// there isn't.
LDHT::Client* LanguageModelLDHT::getClientSafe() {
    if (m_client.get() == NULL)
        m_client.reset(initTSSClient());
    return m_client.get();
}

// Do not check that there is a TSS Client instance.
LDHT::Client* LanguageModelLDHT::getClientUnsafe() const {
    return m_client.get();
}

LDHT::Client* LanguageModelLDHT::initTSSClient() {
    std::ifstream config_file(m_configPath.c_str());
    std::string ldht_config_path;
    getline(config_file, ldht_config_path);
    std::string ldhtlm_config_path;
    getline(config_file, ldhtlm_config_path);

    LDHT::FactoryCollection* factory_collection =
            LDHT::FactoryCollection::createDefaultFactoryCollection();

    LDHT::Client* client;
    client = new LDHT::ClientLocal();
    //client = new LDHT::Client();
    client->fromXmlFiles(*factory_collection,<commit_msg>Use networked implementation of LDHT instead of the local version.<commit_after>//
// <NAME> <<EMAIL>>
//

#include "LM/Base.h"
#include "LM/LDHT.h"
#include "../FFState.h"
#include "../TypeDef.h"
#include "../Hypothesis.h"

#include <LDHT/Client.h>
#include <LDHT/ClientLocal.h>
#include <LDHT/NewNgram.h>
#include <LDHT/FactoryCollection.h>

#include <boost/thread/tss.hpp>

namespace Moses {

struct LDHTLMState : public FFState {
  LDHT::NewNgram gram_fingerprints;

  int Compare(const FFState& uncast_other) const {
    const LDHTLMState &other = static_cast<const LDHTLMState&>(uncast_other);
    return gram_fingerprints.compareMoses(other.gram_fingerprints);
  }

  void copyFrom(const LDHTLMState& other) {
    gram_fingerprints.copyFrom(other.gram_fingerprints);
  }
};

class LanguageModelLDHT : public LanguageModel {
public:
    LanguageModelLDHT();
    LanguageModelLDHT(const std::string& path,
                      ScoreIndexManager& manager,
                      FactorType factorType);
    LanguageModelLDHT(ScoreIndexManager& manager,
                      LanguageModelLDHT& copyFrom);
    std::string GetScoreProducerDescription(unsigned) const {
      std::ostringstream oss;
      oss << "LM_" << LDHT::NewNgram::k_max_order << "gram";
      return oss.str();
    }
    LDHT::Client* getClientUnsafe() const;
    LDHT::Client* getClientSafe();
    LDHT::Client* initTSSClient();
    virtual ~LanguageModelLDHT();
    virtual LanguageModel* Duplicate(
            ScoreIndexManager& scoreIndexManager) const;
    virtual void InitializeBeforeSentenceProcessing();
    virtual void CleanUpAfterSentenceProcessing();
    virtual const FFState* EmptyHypothesisState(const InputType& input) const;
    virtual bool Useable(const Phrase& phrase) const;
    virtual void CalcScore(const Phrase& phrase,
                           float& fullScore,
                           float& ngramScore,
                           std::size_t& oovCount) const;
    FFState* Evaluate(const Hypothesis& hypo,
                      const FFState* input_state,
                      ScoreComponentCollection* score_output) const;
    FFState* EvaluateChart(const ChartHypothesis& hypo,
                           int featureID,
                           ScoreComponentCollection* accumulator) const;

protected:
    boost::thread_specific_ptr<LDHT::Client> m_client;
    std::string m_configPath;
    FactorType m_factorType;

};

LanguageModel* ConstructLDHTLM(const std::string& path,
                               ScoreIndexManager& manager,
                               FactorType factorType) {
    return new LanguageModelLDHT(path, manager, factorType);
}

LanguageModelLDHT::LanguageModelLDHT() : LanguageModel(), m_client(NULL) {
    m_enableOOVFeature = false;
}

LanguageModelLDHT::LanguageModelLDHT(ScoreIndexManager& manager,
                                     LanguageModelLDHT& copyFrom) {
    //m_client = copyFrom.m_client;
    m_factorType = copyFrom.m_factorType;
    m_configPath = copyFrom.m_configPath;
    Init(manager);
}

LanguageModelLDHT::LanguageModelLDHT(const std::string& path,
                                     ScoreIndexManager& manager,
                                     FactorType factorType)
                                            : m_factorType(factorType) {
    m_configPath = path;
    Init(manager);
}

LanguageModelLDHT::~LanguageModelLDHT() {
    // TODO(wilson): should cleanup for each individual thread.
    delete getClientSafe();
}

LanguageModel* LanguageModelLDHT::Duplicate(
            ScoreIndexManager& scoreIndexManager) const {
    return NULL;
}

// Check that there is a TSS Client instance, and instantiate one if
// there isn't.
LDHT::Client* LanguageModelLDHT::getClientSafe() {
    if (m_client.get() == NULL)
        m_client.reset(initTSSClient());
    return m_client.get();
}

// Do not check that there is a TSS Client instance.
LDHT::Client* LanguageModelLDHT::getClientUnsafe() const {
    return m_client.get();
}

LDHT::Client* LanguageModelLDHT::initTSSClient() {
    std::ifstream config_file(m_configPath.c_str());
    std::string ldht_config_path;
    getline(config_file, ldht_config_path);
    std::string ldhtlm_config_path;
    getline(config_file, ldhtlm_config_path);

    LDHT::FactoryCollection* factory_collection =
            LDHT::FactoryCollection::createDefaultFactoryCollection();

    LDHT::Client* client;
    //client = new LDHT::ClientLocal();
    client = new LDHT::Client();
    client->fromXmlFiles(*factory_collection,
{'fraction_non_alphanumeric': 0.10056547289056436, 'fraction_numerical': 0.0, 'mean_word_length': 3.003550821127386, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5152782', 'n_tokens_mistral': 2677, 'n_tokens_neox': 2559, 'n_words': 540}