<commit_before>#include <iostream>
#include <sstream>
#include <fstream>
#include <iomanip>
#include <string>
#include <map>
#include <set>

// Loriano: let's try Armadillo quick code 
#include <armadillo>
#include <cassert>

#include <getopt.h>
#include <unistd.h>
#include <alloca.h>

#include <pcafitter.hpp>
#include <pcaffunctype.hpp>

#define MINDIMLINIT 25

// lstorchi: basic quick code to generate PCA constants

void usage (char * name)
{
  std::cerr << "usage: " << name << " [options] coordinatesfile " << std::endl;
  std::cerr << std::endl;
  std::cerr << " -h, --help                 : display this help and exit" << std::endl;
  std::cerr << " -V, --verbose              : verbose option on" << std::endl;
  std::cerr << " -v, --version              : print version and exit" << std::endl;
  std::cerr << " -f, --fast                 : do not perfomr pca only diag matrix" << std::endl;
  std::cerr << " -i, --seg-id               : use SegId instead of coordinates" << std::endl;
  std::cerr << " -s, --bigger-sub-tower     : use values of the bigger sub-tower" << std::endl;
  std::cerr << "                              (connot be used with bigger-subladder)" << std::endl;
  std::cerr << " -l, --bigger-sub-sub-tower : use values of the bigger sub-sub-tower " << std::endl;
  std::cerr << "                              (connot be used with bigger-subsector)" << std::endl;
  std::cerr << " -a, --all-sub-tower        : generate the constants for all subsectors" << std::endl;
  std::cerr << " -r, --all-sub-sub-tower    : generate the constants for all subladders" << std::endl;
  std::cerr << " -j, --jump-tracks          : generate the constants using only even tracks" << std::endl;
  std::cerr << " -p, --dump-allcoords       : dump all stub coordinates to a file" << std::endl;

  exit(1);
}

void perform_main_computation (const bool fast, const bool verbose, 
    const arma::mat & coord, const arma::mat & param, 
    const std::string & cfname, const std::string & qfname,
    pca::pcafitter & fitter)
{
  // ordered 
  arma::vec eigval;
  // by row or by column ?
  arma::mat eigvec;

  if (!fast)
  {
    std::cout << "Perform PCA " << std::endl;
    // projection 
    arma::mat score;
  
    arma::princomp(eigvec, score, eigval, coord);
    std::cout << score.n_rows << " " << score.n_cols << std::endl;
    
    std::cout << "Writing Scoreplot" << std::endl;
    std::ofstream myfilesc("scoreplot.txt");
    
    for (int i=0; i<(int)score.n_rows; ++i)
    { 
      myfilesc << score(i,0)  << " "
               << score(i,1) << " " << score(i,2) << std::endl;
    
      if (verbose)
      {
        double mainr = 0.0e0;
        for (int j=1; j<5; ++j)
          mainr += score(i,j) * score(i,j);
        
        double residue = 0.0;
        for (int j=5; j<fitter.get_coordim(); ++j)
          residue += score(i,j) * score(i,j);
        
        std::cout << "Track " << i+1 << " residue " << residue <<
                     " mainr " << mainr << std::endl;
      }
    }
    
    myfilesc.close();
  }
  else
  {
    std::cout << "Compute correlation mtx" << std::endl;
    arma::mat coordm = arma::zeros<arma::mat>(fitter.get_coordim());
    arma::mat hca = arma::zeros<arma::mat>(fitter.get_coordim(),
        fitter.get_coordim());
    arma::vec eigvaltmp = arma::zeros<arma::vec>(fitter.get_coordim());

    eigvec = arma::zeros<arma::mat>(fitter.get_coordim(),
        fitter.get_coordim());
    eigval = arma::zeros<arma::vec>(fitter.get_coordim());

    hca = arma::cov(coord);

    /* double check 
    double sum = 1.0e0;
    for (int l=0; l<(int)coord.n_rows; ++l) 
    {
      sum += 1.0e0;
      for (int i=0; i<fitter.get_coordim(); ++i)
        coordm(i) += (coord(l,i)-coordm(i))/sum;
    
      for (int i=0; i<fitter.get_coordim(); ++i)
      {
        for (int j=0; j<fitter.get_coordim(); ++j)
        {
          hca(i,j) += ((coord(l,i) - coordm(i))*
                       (coord(l,j) - coordm(j))-
                       (sum-1.0e0)*hca(i,j)/sum)/(sum-1.0e0);
        }
      }
    }
    */

    std::cout << "Eigensystem" << std::endl;
    arma::eig_sym(eigvaltmp, eigvec, hca);
 
    for (int i=0; i<fitter.get_coordim(); ++i)
      eigval(i) = eigvaltmp(fitter.get_coordim()-i-1);
  }


  double totval = 0.0e0;
  for (int i=0; i<fitter.get_coordim(); ++i)
    totval += eigval(i);

  std::cout << "Eigenvalues: " << std::endl;
  double totvar = 0.0e0; 
  for (int i=0; i<fitter.get_coordim(); ++i)
  {
    if (i < fitter.get_paramdim())
      totvar += 100.0e0*(eigval(i)/totval);

    std::cout << i+1 << " ==> " << 100.0e0*(eigval(i)/totval) 
              << "% value: " << eigval(i) <<  std::endl;
  }
  std::cout << "PARAMDIM eigenvalues: " << totvar << std::endl;

  std::cout << fitter.get_paramdim() << " X " << fitter.get_coordim() << std::endl;

  arma::mat cmtx = arma::zeros<arma::mat>(fitter.get_paramdim(),
      fitter.get_coordim());
  arma::rowvec q = arma::zeros<arma::rowvec>(fitter.get_paramdim());

  std::cout << "Compute PCA constants " << std::endl;
  fitter.compute_pca_constants (param,
      coord, cmtx, q);

  std::cout << "Write constant to file" << std::endl;
  pca::write_armmat(cfname.c_str(), cmtx);
  pca::write_armvct(qfname.c_str(), q);
}

int main (int argc, char ** argv)
{
  pca::pcafitter fitter; 

  bool fast = false;
  bool verbose = false;
  bool selectsubsector = true;
  bool selectsubladder = false;
  bool useallsubsectors = false;
  bool useallsubladders = false;
  bool useonlyeven = false;
  bool usesegid = false;
  bool printallcoords = false;

  while (1)
  {<commit_msg>minor<commit_after>#include <iostream>
#include <sstream>
#include <fstream>
#include <iomanip>
#include <string>
#include <map>
#include <set>

// Loriano: let's try Armadillo quick code 
#include <armadillo>
#include <cassert>

#include <getopt.h>
#include <unistd.h>
#include <alloca.h>

#include <pcafitter.hpp>
#include <pcaffunctype.hpp>

#define MINDIMLINIT 25

// lstorchi: basic quick code to generate PCA constants

void usage (char * name)
{
  std::cerr << "usage: " << name << " [options] coordinatesfile " << std::endl;
  std::cerr << std::endl;
  std::cerr << " -h, --help                 : display this help and exit" << std::endl;
  std::cerr << " -V, --verbose              : verbose option on" << std::endl;
  std::cerr << " -v, --version              : print version and exit" << std::endl;
  std::cerr << " -f, --fast                 : do not perfomr pca only diag matrix" << std::endl;
  std::cerr << " -i, --seg-id               : use SegId instead of coordinates" << std::endl;
  std::cerr << " -s, --bigger-sub-tower     : use values of the bigger sub-tower" << std::endl;
  std::cerr << "                              (connot be used with bigger-subladder)" << std::endl;
  std::cerr << " -l, --bigger-sub-sub-tower : use values of the bigger sub-sub-tower " << std::endl;
  std::cerr << "                              (connot be used with bigger-subsector)" << std::endl;
  std::cerr << " -a, --all-sub-tower        : generate the constants for all subsectors" << std::endl;
  std::cerr << " -r, --all-sub-sub-tower    : generate the constants for all subladders" << std::endl;
  std::cerr << " -j, --jump-tracks          : generate the constants using only even tracks" << std::endl;
  std::cerr << " -p, --dump-allcoords       : dump all stub coordinates to a file" << std::endl;

  exit(1);
}

void perform_main_computation (const bool fast, const bool verbose, 
    const arma::mat & coord, const arma::mat & param, 
    const std::string & cfname, const std::string & qfname,
    pca::pcafitter & fitter)
{
  // ordered 
  arma::vec eigval;
  // by row or by column ?
  arma::mat eigvec;

  if (!fast)
  {
    std::cout << "Perform PCA " << std::endl;
    // projection 
    arma::mat score;
  
    arma::princomp(eigvec, score, eigval, coord);
    std::cout << score.n_rows << " " << score.n_cols << std::endl;
    
    std::cout << "Writing Scoreplot" << std::endl;
    std::ofstream myfilesc("scoreplot.txt");
    
    for (int i=0; i<(int)score.n_rows; ++i)
    { 
      myfilesc << score(i,0)  << " "
               << score(i,1) << " " << score(i,2) << std::endl;
    
      if (verbose)
      {
        double mainr = 0.0e0;
        for (int j=1; j<5; ++j)
          mainr += score(i,j) * score(i,j);
        
        double residue = 0.0;
        for (int j=5; j<fitter.get_coordim(); ++j)
          residue += score(i,j) * score(i,j);
        
        std::cout << "Track " << i+1 << " residue " << residue <<
                     " mainr " << mainr << std::endl;
      }
    }
    
    myfilesc.close();
  }
  else
  {
    std::cout << "Compute correlation mtx" << std::endl;
    arma::mat coordm = arma::zeros<arma::mat>(fitter.get_coordim());
    arma::mat hca = arma::zeros<arma::mat>(fitter.get_coordim(),
        fitter.get_coordim());
    arma::vec eigvaltmp = arma::zeros<arma::vec>(fitter.get_coordim());

    eigvec = arma::zeros<arma::mat>(fitter.get_coordim(),
        fitter.get_coordim());
    eigval = arma::zeros<arma::vec>(fitter.get_coordim());

    hca = arma::cov(coord);

    /* double check 
    double sum = 1.0e0;
    for (int l=0; l<(int)coord.n_rows; ++l) 
    {
      sum += 1.0e0;
      for (int i=0; i<fitter.get_coordim(); ++i)
        coordm(i) += (coord(l,i)-coordm(i))/sum;
    
      for (int i=0; i<fitter.get_coordim(); ++i)
      {
        for (int j=0; j<fitter.get_coordim(); ++j)
        {
          hca(i,j) += ((coord(l,i) - coordm(i))*
                       (coord(l,j) - coordm(j))-
                       (sum-1.0e0)*hca(i,j)/sum)/(sum-1.0e0);
        }
      }
    }
    */

    std::cout << "Eigensystem" << std::endl;
    arma::eig_sym(eigvaltmp, eigvec, hca);
 
    for (int i=0; i<fitter.get_coordim(); ++i)
      eigval(i) = eigvaltmp(fitter.get_coordim()-i-1);
  }


  double totval = 0.0e0;
  for (int i=0; i<fitter.get_coordim(); ++i)
    totval += eigval(i);

  std::cout << "Eigenvalues: " << std::endl;
  double totvar = 0.0e0; 
  for (int i=0; i<fitter.get_coordim(); ++i)
  {
    if (i < fitter.get_paramdim())
      totvar += 100.0e0*(eigval(i)/totval);

    std::cout << i+1 << " ==> " << 100.0e0*(eigval(i)/totval) 
              << "% value: " << eigval(i) <<  std::endl;
  }
  std::cout << "PARAMDIM eigenvalues: " << totvar << std::endl;

  std::cout << fitter.get_paramdim() << " X " << fitter.get_coordim() << std::endl;

  arma::mat cmtx = arma::zeros<arma::mat>(fitter.get_paramdim(),
      fitter.get_coordim());
  arma::rowvec q = arma::zeros<arma::rowvec>(fitter.get_paramdim());

  std::cout << "Compute PCA constants " << std::endl;
  fitter.compute_pca_constants (param,
      coord, cmtx, q, verbose);

  std::cout << "Write constant to file" << std::endl;
  pca::write_armmat(cfname.c_str(), cmtx);
  pca::write_armvct(qfname.c_str(), q);
}

int main (int argc, char ** argv)
{
  pca::pcafitter fitter; 

  bool fast = false;
  bool verbose = false;
  bool selectsubsector = true;
  bool selectsubladder = false;
  bool useallsubsectors = false;
  bool useallsubladders = false;
  bool useonlyeven = false;
  bool usesegid = false;
  bool printallcoords = false;

  while (1)
  {
{'fraction_non_alphanumeric': 0.18449030644152595, 'fraction_numerical': 0.009648887697668185, 'mean_word_length': 2.6179702650290886, 'pattern_counts': {'":': 0, '<': 367, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1866353', 'n_tokens_mistral': 4327, 'n_tokens_neox': 4162, 'n_words': 907}