<commit_before>/* Copyright (c) 2012, <NAME> <<EMAIL>>
 * Licensed under the MIT license. See LICENSE.txt or 
 * http://www.opensource.org/licenses/mit-license.php */

#pragma once

#include "baseMeasure.hpp"
#include "probabilityHelpers.hpp"

#include <stddef.h>
#include <stdint.h>
#include <typeinfo>

#include <armadillo>

using namespace std;
using namespace arma;

template <class U>
class HDP // : public DP<U>
{
  public:
    HDP(const BaseMeasure<U>& base, double alpha, double omega)
      : mH(base), mAlpha(alpha), mOmega(omega)
    { };

    ~HDP()
    {	};

    /* 
     * interface mainly for python
     * @return the index of the added x_i in the set of all documents
     */
    uint32_t addDoc(const Mat<U>& x_i)
    {
      mX.push_back(x_i);
      return mX.size()-1; 
    };

    // interface mainly for python
    uint32_t addHeldOut(const Mat<U>& x_i)
    {
      //cout<<"added heldout doc with "<<x_i.size()<<" words"<<endl;
      uint32_t N = x_i.n_cols;
      
      //DONE: this shuffle might not be working
      cout<<"x_i: "<<x_i.cols(0,10)<<endl;
      Mat<U> x_s = shuffle(x_i,1); // suffle the columns (words)
      cout<<"x_s: "<<x_s.cols(0,10)<<endl;
      Mat<U> x_te = x_s.cols(0,(N/2)-1) ; // select first half to train on
      Mat<U> x_ho = x_s.cols(N/2,N-1) ; // select second half as held out

      cout<<"x_te: "<<x_te.n_rows<<"x"<<x_te.n_cols<<endl;
      cout<<"x_ho: "<<x_ho.n_rows<<"x"<<x_ho.n_cols<<endl;
      
  
      mX_te.push_back(x_te); // on half of the data we train as usual to get a topic model for the document
      mX_ho.push_back(x_ho); // held out data to evaluate the perplexity
      return mX_ho.size();
    };

    //virtual Row<double> logP_w(uint32_t d) const=0;


    double perplexity(const Mat<U>& x_ho, const Row<double>& logP) const
    {
      assert(x_ho.n_rows==1);

      uint32_t N = x_ho.n_cols;
      double perp = 0.0;
      for (uint32_t i=0; i<N; ++i){
        //cout<<"c_z_n = "<<c[z[w]]<<" z_n="<<z[w]<<" w="<<w<<" N="<<N<<" x_w="<<x_ho[w]<<" topics.shape="<<topics.n_rows<<" "<<topics.n_cols;
        cout<<"x_ho_i="<<x_ho(i)<<"; logP_x_ho_i="<<logP(x_ho(i))<<endl;
          perp -= logP(x_ho(i));
        //cout<<"w="<<w<<"\tx_ho_w="<<x_ho[w]<<"\tlogP="<<logP[w]<<"\tperp+="<<-double(x_ho[w])*logP[w]<<endl;
      } cout<<endl;
      perp /= double(N);
      perp /= log(2.0); // since it is log base 2 in the perplexity formulation!
      perp = pow(2.0,perp);
      cout<<"perp="<<perp<<endl;

      return perp;
    };

protected:

    const BaseMeasure<U>& mH; // base measure
    double mAlpha; 
    double mOmega;
    vector<Mat<U> > mX; // training data
    vector<Mat<U> > mX_te; //  test data
    vector<Mat<U> > mX_ho; //  held out data

private:

};

template <class U>
class HDP_var_base 
{
  public:

    HDP_var_base(uint32_t K, uint32_t T, uint32_t Nw)
      : mK(K), mT(T), mNw(Nw)
    {};
//    void getA(Col<double>& a)
//    {
//      a=mA.col(0);
//    };
//
//    void getB(Col<double>& b)
//    {
//      b=mA.col(1);
//    };
//
//    bool getLambda(Col<double>& lambda, uint32_t k)
//    {
//      if(mLambda.n_rows > 0 && k < mLambda.n_rows)
//      {
//        lambda=mLambda.row(k).t();
//        return true;
//      }else{
//        return false;
//      }
//    };
//<commit_msg>now revmoving a lot of commented out old stuff that was duplicated<commit_after>/* Copyright (c) 2012, <NAME> <<EMAIL>>
 * Licensed under the MIT license. See LICENSE.txt or 
 * http://www.opensource.org/licenses/mit-license.php */

#pragma once

#include "baseMeasure.hpp"
#include "probabilityHelpers.hpp"

#include <stddef.h>
#include <stdint.h>
#include <typeinfo>

#include <armadillo>

using namespace std;
using namespace arma;

template <class U>
class HDP // : public DP<U>
{
  public:
    HDP(const BaseMeasure<U>& base, double alpha, double omega)
      : mH(base), mAlpha(alpha), mOmega(omega)
    { };

    ~HDP()
    {	};

    /* 
     * interface mainly for python
     * @return the index of the added x_i in the set of all documents
     */
    uint32_t addDoc(const Mat<U>& x_i)
    {
      mX.push_back(x_i);
      return mX.size()-1; 
    };

    // interface mainly for python
    uint32_t addHeldOut(const Mat<U>& x_i)
    {
      //cout<<"added heldout doc with "<<x_i.size()<<" words"<<endl;
      uint32_t N = x_i.n_cols;
      
      //DONE: this shuffle might not be working
      cout<<"x_i: "<<x_i.cols(0,10)<<endl;
      Mat<U> x_s = shuffle(x_i,1); // suffle the columns (words)
      cout<<"x_s: "<<x_s.cols(0,10)<<endl;
      Mat<U> x_te = x_s.cols(0,(N/2)-1) ; // select first half to train on
      Mat<U> x_ho = x_s.cols(N/2,N-1) ; // select second half as held out

      cout<<"x_te: "<<x_te.n_rows<<"x"<<x_te.n_cols<<endl;
      cout<<"x_ho: "<<x_ho.n_rows<<"x"<<x_ho.n_cols<<endl;
      
  
      mX_te.push_back(x_te); // on half of the data we train as usual to get a topic model for the document
      mX_ho.push_back(x_ho); // held out data to evaluate the perplexity
      return mX_ho.size();
    };

    //virtual Row<double> logP_w(uint32_t d) const=0;


    double perplexity(const Mat<U>& x_ho, const Row<double>& logP) const
    {
      assert(x_ho.n_rows==1);

      uint32_t N = x_ho.n_cols;
      double perp = 0.0;
      for (uint32_t i=0; i<N; ++i){
        //cout<<"c_z_n = "<<c[z[w]]<<" z_n="<<z[w]<<" w="<<w<<" N="<<N<<" x_w="<<x_ho[w]<<" topics.shape="<<topics.n_rows<<" "<<topics.n_cols;
        cout<<"x_ho_i="<<x_ho(i)<<"; logP_x_ho_i="<<logP(x_ho(i))<<endl;
          perp -= logP(x_ho(i));
        //cout<<"w="<<w<<"\tx_ho_w="<<x_ho[w]<<"\tlogP="<<logP[w]<<"\tperp+="<<-double(x_ho[w])*logP[w]<<endl;
      } cout<<endl;
      perp /= double(N);
      perp /= log(2.0); // since it is log base 2 in the perplexity formulation!
      perp = pow(2.0,perp);
      cout<<"perp="<<perp<<endl;

      return perp;
    };

protected:

    const BaseMeasure<U>& mH; // base measure
    double mAlpha; 
    double mOmega;
    vector<Mat<U> > mX; // training data
    vector<Mat<U> > mX_te; //  test data
    vector<Mat<U> > mX_ho; //  held out data

private:

};

template <class U>
class HDP_var_base 
{
  public:

    HDP_var_base(uint32_t K, uint32_t T, uint32_t Nw)
      : mK(K), mT(T), mNw(Nw)
    {};

//    void getA(Col<double>& a)
//    {
//      a=mA.col(0);
//    };
//
//    void getB(Col<double>& b)
//    {
//      b=mA.col(1);
//    };
//
//    bool getLambda(Col<double>& lambda, uint32_t k)
//    {
//      if(mLambda.n_rows > 0 && k < mLambda.n_rows)
//      {
//        lambda=mLambda.row(k).t();
//        return true;
//      }else{
//        return false;
//      }
//    };
//
{'fraction_non_alphanumeric': 0.19014719134875338, 'fraction_numerical': 0.015019525382997898, 'mean_word_length': 2.525145579671784, 'pattern_counts': {'":': 0, '<': 273, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '297275', 'n_tokens_mistral': 2806, 'n_tokens_neox': 2713, 'n_words': 577}