<commit_before>#include "fastlib/fastlib.h"

const fx_entry_doc kernel_matrix_generator_entries[] = {
  {"data", FX_REQUIRED, FX_STR, NULL,
   " File consists of data points in lines.\n"},
  {"bandwidth", FX_REQUIRED, FX_DOUBLE, NULL,
   " The kernel bandwidth.\n"},
  {"rate", FX_PARAM, FX_INT, NULL,
   " The sampling rate, default = 20.\n"},
  {"output", FX_PARAM, FX_STR, NULL,
   " File to hold output kernel matrix,"
   " default = \"kernel_matrix.txt\".\n"},
  FX_ENTRY_DOC_DONE
};

const fx_submodule_doc kernel_matrix_generator_submodules[] = {
  FX_SUBMODULE_DOC_DONE
};

const fx_module_doc kernel_matrix_generator_doc = {
  kernel_matrix_generator_entries, kernel_matrix_generator_submodules,
  "This is a program generating the kernel matrix of "
  "a set of points.\n"
};

int main(int argc, char *argv[]) {

  // initialize FastExec (parameter handling stuff)
  fx_init(argc, argv, &kernel_matrix_generator_doc);
  
  Matrix references;
  const char *references_file_name = fx_param_str_req(fx_root, "data");
  double bandwidth = fx_param_double_req(fx_root, "bandwidth");
  data::Load(references_file_name, &references);

  printf("nrows = %d ncols = %d\n", references.n_rows(), 
	 references.n_cols());
#define RUNNING 1
#if RUNNING==1  
  // Kernel matrix to be outputted.
  Matrix kernel_matrix;

  // Initialize the kernel.
  GaussianKernel kernel;
  kernel.Init(bandwidth);

  int rate = fx_param_int(fx_root, "rate", 20);
  int n_cols = (references.n_cols()-1)/rate+1;
  kernel_matrix.Init(n_cols, n_cols);
  for(index_t r = 0; r < references.n_cols(); r+=rate) {
    const double *r_col = references.GetColumnPtr(r);

    for(index_t q = 0; q < references.n_cols(); q+=rate) {
      const double *q_col = references.GetColumnPtr(q);
      double dsqd = la::DistanceSqEuclidean(references.n_rows(), q_col,
					    r_col);
      kernel_matrix.set(q/rate, r/rate, kernel.EvalUnnormOnSq(dsqd));
    }
  }
<commit_msg>add polynomial kernel matrix generation<commit_after>#include "fastlib/fastlib.h"

const fx_entry_doc kernel_matrix_generator_entries[] = {
  {"data", FX_REQUIRED, FX_STR, NULL,
   " File consists of data points in lines.\n"},
  {"kernel", FX_REQUIRED, FX_STR, NULL,
   " The kernel type: gaussian / polynomial.\n"},
  {"bandwidth", FX_PARAM, FX_DOUBLE, NULL,
   " The gaussian kernel bandwidth, default = 1.\n"},
  {"degree", FX_PARAM, FX_DOUBLE, NULL,
   " The polynomial kernel degree, default = 1.\n"},
  {"rate", FX_PARAM, FX_INT, NULL,
   " The sampling rate, default = 20 choose 1.\n"},
  {"output", FX_PARAM, FX_STR, NULL,
   " File to hold output kernel matrix,"
   " default = \"kernel_matrix.txt\".\n"},
  FX_ENTRY_DOC_DONE
};

const fx_submodule_doc kernel_matrix_generator_submodules[] = {
  FX_SUBMODULE_DOC_DONE
};

const fx_module_doc kernel_matrix_generator_doc = {
  kernel_matrix_generator_entries, kernel_matrix_generator_submodules,
  "This is a program generating the kernel matrix of "
  "a set of points.\n"
};

void gen_gaussian_kernel(const Matrix& ref, Matrix* kernel_matrix,
			 int rate) {
  double bandwidth = fx_param_double(fx_root, "bandwidth", 1);

  // Initialize the kernel.
  GaussianKernel kernel;
  kernel.Init(bandwidth);

  int n_cols = (ref.n_cols()-1)/rate+1;
  kernel_matrix->Init(n_cols, n_cols);
  for(index_t r = 0; r < ref.n_cols(); r+=rate) {
    const double *r_col = ref.GetColumnPtr(r);
    
    for(index_t q = 0; q < ref.n_cols(); q+=rate) {
      const double *q_col = ref.GetColumnPtr(q);
      double dsqd = la::DistanceSqEuclidean(ref.n_rows(), q_col,
					    r_col);
      kernel_matrix->set(q/rate, r/rate, kernel.EvalUnnormOnSq(dsqd));
    }
  }
}

void gen_polynomial_kernel(const Matrix& ref, Matrix* kernel_matrix,
			   int rate) {
  double degree = fx_param_double(fx_root, "degree", 1);

  int n_cols = (ref.n_cols()-1)/rate+1;
  kernel_matrix->Init(n_cols, n_cols);
  for(index_t r = 0; r < ref.n_cols(); r+=rate) {
    const double *r_col = ref.GetColumnPtr(r);
    
    for(index_t q = 0; q < ref.n_cols(); q+=rate) {
      const double *q_col = ref.GetColumnPtr(q);
      double dsqd = la::Dot(ref.n_rows(), q_col, r_col) + 1;
      kernel_matrix->set(q/rate, r/rate, pow(dsqd, degree));
    }
  }
}

int main(int argc, char *argv[]) {

  // initialize FastExec (parameter handling stuff)
  fx_init(argc, argv, &kernel_matrix_generator_doc);
  
  Matrix references;
  const char *references_file_name = fx_param_str_req(fx_root, "data");
  data::Load(references_file_name, &references);

  printf("nrows = %d ncols = %d\n", references.n_rows(), 
	 references.n_cols());

  const char* kernel_type = fx_param_str_req(fx_root, "kernel");
  int rate = fx_param_int(fx_root, "rate", 20);

#define RUNNING 1
#if RUNNING==1  

  Matrix kernel_matrix;
  if (strcmp(kernel_type, "gaussian") == 0) 
    gen_gaussian_kernel(references, &kernel_matrix, rate);
  else if (strcmp(kernel_type, "polynomial") == 0) 
    gen_polynomial_kernel(references, &kernel_matrix, rate);

  /*
  // Kernel matrix to be outputted.
  Matrix kernel_matrix;

  // Initialize the kernel.
  GaussianKernel kernel;
  kernel.Init(bandwidth);

  int n_cols = (references.n_cols()-1)/rate+1;
  kernel_matrix.Init(n_cols, n_cols);
  for(index_t r = 0; r < references.n_cols(); r+=rate) {
    const double *r_col = references.GetColumnPtr(r);

    for(index_t q = 0; q < references.n_cols(); q+=rate) {
      const double *q_col = references.GetColumnPtr(q);
      double dsqd = la::DistanceSqEuclidean(references.n_rows(), q_col,
					    r_col);
      kernel_matrix.set(q/rate, r/rate, kernel.EvalUnnormOnSq(dsqd));
    }
  }
  */

{'fraction_non_alphanumeric': 0.1368665239114918, 'fraction_numerical': 0.006423982869379015, 'mean_word_length': 4.1422018348623855, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4041328', 'n_tokens_mistral': 2128, 'n_tokens_neox': 2028, 'n_words': 456}