<commit_before>#include "helper.hpp"

#include <iostream>

void ComputeLowRank
(EMatrix& UMat, EMatrix& VMat, const EMatrix& A) {

#ifdef DEBUG
  std::cout << "  Form the low rank block ..." << std::endl;
#endif

  int nRow = A.rows();
  int nCol = A.cols();
  int k = nRow > nCol ? nCol : nRow; // the smaller
  EMatrix Ufull = EMatrix::Zero( nRow, k );
  EMatrix Vfull = EMatrix::Zero( k, nCol );
  int rank = 0;
  for (int c=0; c<nCol; c++) {
    for (int r=0; r<nRow; r++) {
      if (A(r,c) != 0) {
#ifdef DEBUG
	assert( rank < k );
#endif
	Ufull(r,rank) = A(r,c);
	Vfull(rank,c) = 1;
	rank++;
      }
    }
  }
  if (rank>0) {
    UMat = Ufull.leftCols(rank);
    VMat = Vfull.topRows(rank);
  }
  else {
    // zero vectors
    UMat = Ufull.leftCols(1);
    VMat = Vfull.topRows(1);
  }
#ifdef DEBUG
  std::cout << "Error for low rank approximation: "
	    << (A - UMat*VMat).norm()
	    << std::endl;
#endif
}

void ComputeLowRank_SVD
(EMatrix& UMat, EMatrix& VMat, const EMatrix& A, double eps) {
  
#ifdef DEBUG
  std::cout << "  Form the low rank block ..." << std::endl;
#endif

  Eigen::JacobiSVD<EMatrix> svd(A, Eigen::ComputeThinU
				|  Eigen::ComputeThinV);
  Eigen::VectorXd S = svd.singularValues();
  EMatrix U = svd.matrixU();
  EMatrix V = svd.matrixV();

  int i=0;
  for (; i<S.size(); i++) {
    if (S(i+1)/S(i) > eps)
      U.col(i) *= S(i);
    else
      break;
  }
  U.col(i) *= S(i);
  UMat = U.leftCols(i+1);
  VMat = V.leftCols(i+1).transpose();
#ifdef DEBUG
  std::cout << "Error for low rank approximation: "
	    << (A - UMat*VMat).norm()
	    << std::endl;
#endif
}

// compute the partial sum of the input array
//  e.g. given : 1, 2, 3
//       ouput : 0, 1, 3, 6
// (assume 0 at the end of the array, so for the above example,
//  the actual input should be : 1, 2, 3, 0
void PartialSum( std::vector<int>& vec) {
  int sum  = 0;
  int size = vec.size();
  int i = 0;
  for (; i<size-1; i++) {
    int tmp = vec[i];
    vec[i] = sum;
    sum += tmp;
  }
  vec[i] = sum;
}


// The (global) low rank prepresentation for 2x2 low rank blocks
//    -----------------------
//    |          |          |
//    |  u0*v0T  |  u1*v1T  |
//    |          |          |
//    -----------------------<commit_msg>debug ComputeLowRank_SVD() to handle full rank matrices<commit_after>#include "helper.hpp"

#include <iostream>

void ComputeLowRank
(EMatrix& UMat, EMatrix& VMat, const EMatrix& A) {

#ifdef DEBUG
  std::cout << "  Form the low rank block ..." << std::endl;
#endif

  int nRow = A.rows();
  int nCol = A.cols();
  int k = nRow > nCol ? nCol : nRow; // the smaller
  EMatrix Ufull = EMatrix::Zero( nRow, k );
  EMatrix Vfull = EMatrix::Zero( k, nCol );
  int rank = 0;
  for (int c=0; c<nCol; c++) {
    for (int r=0; r<nRow; r++) {
      if (A(r,c) != 0) {
#ifdef DEBUG
	assert( rank < k );
#endif
	Ufull(r,rank) = A(r,c);
	Vfull(rank,c) = 1;
	rank++;
      }
    }
  }
  if (rank>0) {
    UMat = Ufull.leftCols(rank);
    VMat = Vfull.topRows(rank);
  }
  else {
    // zero vectors
    UMat = Ufull.leftCols(1);
    VMat = Vfull.topRows(1);
  }
#ifdef DEBUG
  std::cout << "Error for low rank approximation: "
	    << (A - UMat*VMat).norm()
	    << std::endl;
#endif
}

void ComputeLowRank_SVD
(EMatrix& UMat, EMatrix& VMat, const EMatrix& A, double eps) {
  
#ifdef DEBUG
  std::cout << "  Form the low rank block ..." << std::endl;
  std::cout << A << std::endl;
#endif

  Eigen::JacobiSVD<EMatrix> svd(A, Eigen::ComputeThinU
				|  Eigen::ComputeThinV);
  Eigen::VectorXd S = svd.singularValues();
  EMatrix U = svd.matrixU();
  EMatrix V = svd.matrixV();

  // handle (nearly) zero matrix
  if ( S(0) < eps )
    return;
  
  U.col(0) *= S(0);
  int i=1;
  for (; i<S.size(); i++) {
    if (S(i)/S(i-1) > eps)
      U.col(i) *= S(i);
    else
      break;
  }
  UMat = U.leftCols(i);
  VMat = V.leftCols(i).transpose();
  
#ifdef DEBUG
  std::cout << "Matrix size : " << S.size()
	    << " rank : " << i
	    << std::endl;
  std::cout << "Error for low rank approximation: "
	    << (A - UMat*VMat).norm()
	    << std::endl;
#endif
}

// compute the partial sum of the input array
//  e.g. given : 1, 2, 3
//       ouput : 0, 1, 3, 6
// (assume 0 at the end of the array, so for the above example,
//  the actual input should be : 1, 2, 3, 0
void PartialSum( std::vector<int>& vec) {
  int sum  = 0;
  int size = vec.size();
  int i = 0;
  for (; i<size-1; i++) {
    int tmp = vec[i];
    vec[i] = sum;
    sum += tmp;
  }
  vec[i] = sum;
}


// The (global) low rank prepresentation for 2x2 low rank blocks
//    -----------------------
//    |          |          |
//    |  u0*v0T  |  u1*v1T  |
//    |          |          |
//    -----------------------
{'fraction_non_alphanumeric': 0.19838401020625132, 'fraction_numerical': 0.014246225813310653, 'mean_word_length': 2.2419021364576155, 'pattern_counts': {'":': 0, '<': 74, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1147642', 'n_tokens_mistral': 1960, 'n_tokens_neox': 1886, 'n_words': 504}