<commit_before>// Adapted with permission from code by <NAME>
// (https://github.com/edrumwri).

#include "MobyLCP.h"

#include <Eigen/LU>
#include <Eigen/SparseCore>
#include <Eigen/SparseLU>

#include <cmath>
#include <iostream>
#include <limits>
#include <memory>
#include <sstream>
#include <vector>

#include "Optimization.h"

namespace {
// TODO sammy these helpers probably have some faster, more Eigen way
// to do this than I've come up with.  This implementation, anyway, is
// definitely slow.  It's an attempt at rewriting some Ravelin
// functionality.  Also, the only reason that the out paramater is a
// MatrixXd is so that this function can resize it.  Runtime
// performance may be improved by allowing the caller to create a
// fixed-size matrix and pass that in instead.
template <typename Derived>
void selectSubMat(const Eigen::MatrixBase<Derived>& in,
                  const std::vector<unsigned>& rows,
                  const std::vector<unsigned>& cols,
                  Eigen::MatrixXd* out) {
  const int num_rows = rows.size();
  const int num_cols = cols.size();
  out->resize(num_rows, num_cols);

  for (int i = 0; i < num_rows; i++) {
    const auto row_in = in.row(rows[i]);
    auto row_out = out->row(i);
    for (int j = 0; j < num_cols; j++) {
      row_out(j) = row_in(cols[j]);
    }
  }
}

// TODO sammy this could also use a more efficient implementation.<commit_msg>Remove a TODO as there seems to be a consensus that it can't really be done.<commit_after>// Adapted with permission from code by <NAME>
// (https://github.com/edrumwri).

#include "MobyLCP.h"

#include <Eigen/LU>
#include <Eigen/SparseCore>
#include <Eigen/SparseLU>

#include <cmath>
#include <iostream>
#include <limits>
#include <memory>
#include <sstream>
#include <vector>

#include "Optimization.h"

namespace {
template <typename Derived>
void selectSubMat(const Eigen::MatrixBase<Derived>& in,
                  const std::vector<unsigned>& rows,
                  const std::vector<unsigned>& cols,
                  Eigen::MatrixXd* out) {
  const int num_rows = rows.size();
  const int num_cols = cols.size();
  out->resize(num_rows, num_cols);

  for (int i = 0; i < num_rows; i++) {
    const auto row_in = in.row(rows[i]);
    auto row_out = out->row(i);
    for (int j = 0; j < num_cols; j++) {
      row_out(j) = row_in(cols[j]);
    }
  }
}

// TODO sammy this could also use a more efficient implementation.
{'fraction_non_alphanumeric': 0.1287940935192781, 'fraction_numerical': 0.0016406890894175555, 'mean_word_length': 3.3321492007104796, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4194813', 'n_tokens_mistral': 821, 'n_tokens_neox': 791, 'n_words': 265}