<commit_before>/*
*   Copyright (c) 2010-2014, MIT Probabilistic Computing Project
*
*   Lead Developers: <NAME> and <NAME>
*   Authors: <NAME>, <NAME>, <NAME>, <NAME>
*   Research Leads: <NAME>, <NAME>
*
*   Licensed under the Apache License, Version 2.0 (the "License");
*   you may not use this file except in compliance with the License.
*   You may obtain a copy of the License at
*
*       http://www.apache.org/licenses/LICENSE-2.0
*
*   Unless required by applicable law or agreed to in writing, software
*   distributed under the License is distributed on an "AS IS" BASIS,
*   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*   See the License for the specific language governing permissions and
*   limitations under the License.
*/
#include "RandomNumberGenerator.h"
#include "utils.h"
//
#include <fstream>      // fstream
#include <boost/tokenizer.hpp>
#include <boost/numeric/ublas/matrix.hpp>
#include <boost/numeric/ublas/matrix_proxy.hpp>
#include <numeric>
#include <algorithm>

using namespace std;
using namespace boost;

using boost::numeric::ublas::project;
using boost::numeric::ublas::matrix;

// FROM runModel_v2.cpp
/////////////////////////////////////////////////////////////////////
// expect a csv file of data
void LoadData(const string& file, matrix<double>& M) {
  ifstream in(file.c_str());
  if (!in.is_open()) return;
  typedef tokenizer< char_separator<char> > Tokenizer;
  char_separator<char> sep(",");

  string line;
  int nrows = 0; 
  int ncols = 0;
  vector<string> vec;

  // get the size first
  while (std::getline(in,line)) {
    Tokenizer tok(line, sep);
    vec.assign(tok.begin(), tok.end());
    ncols = vec.end() - vec.begin();
    nrows++;
  }
  cout << "num rows = "<< nrows << "  num cols = " << ncols << endl;

  // create a matrix to hold data
  matrix<double> Data(nrows, ncols);
  
  // make second pass 
  in.clear();
  in.seekg(0);
  int r = 0;
  while (std::getline(in,line)) {
    Tokenizer tok(line, sep);
    vec.assign(tok.begin(), tok.end());
    unsigned int i = 0;
    for(i=0; i < vec.size() ; i++) {
      Data(r, i) = ::strtod(vec[i].c_str(), 0);
    }
    r++;
  }
  M = Data;
}

bool is_almost(double val1, double val2, double precision) {
  return abs(val1-val2) < precision;
}

// linspace(a, b, n)
//
//      Return a vector of n equidistant points between a and b,
//      inclusive, for n >= 2.  That is,
//
//              [a, a + s, a + 2s, a + 3s, ..., a + (n - 2)s, b],
//
//      where s = (b - a)/(n - 1) is the distance between consecutive
//      points.  If v is the result, then v[0] = a and v[n - 1] = b.
vector<double> linspace(double a, double b, size_t n) {
  assert(2 <= n);
  assert(a <= b);

  vector<double> v(n);
  const double s = (b - a)/(n - 1);

  v[0] = a;
  for (size_t i = 1; i < (n - 1); i++)
    v[i] = a + i*s;
  v[n - 1] = b;

  return v;
}

// log_linspace(a, b, n)
//
//      Return a vector of n `equiratioed' points between a and b,
//      inclusive, for n >= 2.  That is,
//
//              [a, a s, a s^2, a s^3, ..., a s^(n - 2), b],
//
//      where s = (b/a)^1/(n - 1) is the ratio of consecutive points.
//      If v is the result, then v[0] = a and v[n - 1] = b, except in
//      the case described below.
//
//	If a or b is too close to zero to be represented as nonzero,
//	it is first rounded up to the smallest positive normal value.
//	(Analytically, zero makes no sense for either a or b, but
//	values sufficiently close to zero will have been rounded to
//	zero beforehand.)
vector<double> log_linspace(double a, double b, size_t n) {
  assert(2 <= n);
  assert(a <= b);

  a = std::max(a, std::numeric_limits<double>::min());
  b = std::max(b, std::numeric_limits<double>::min());

  vector<double> v(n);
  const double log_a = log(a);
  const double log_r = (log(b) - log_a)/(n - 1);

  v[0] = a;
  for (size_t i = 1; i < (n - 1); i++)
    v[i] = std::min(b, exp(log_a + i*log_r));
  v[n - 1] = b;

  return v;
}

vector<double> std_vector_add(const vector<double>& vec1,
                              const vector<double>& vec2) {
  assert(vec1.size()==vec2.size());<commit_msg>Paranoia: clamp below at a in log_linspace too.<commit_after>/*
*   Copyright (c) 2010-2014, MIT Probabilistic Computing Project
*
*   Lead Developers: <NAME> and <NAME>
*   Authors: <NAME>, <NAME>, <NAME>, <NAME>
*   Research Leads: Vikash Mansinghka, <NAME>
*
*   Licensed under the Apache License, Version 2.0 (the "License");
*   you may not use this file except in compliance with the License.
*   You may obtain a copy of the License at
*
*       http://www.apache.org/licenses/LICENSE-2.0
*
*   Unless required by applicable law or agreed to in writing, software
*   distributed under the License is distributed on an "AS IS" BASIS,
*   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*   See the License for the specific language governing permissions and
*   limitations under the License.
*/
#include "RandomNumberGenerator.h"
#include "utils.h"
//
#include <fstream>      // fstream
#include <boost/tokenizer.hpp>
#include <boost/numeric/ublas/matrix.hpp>
#include <boost/numeric/ublas/matrix_proxy.hpp>
#include <numeric>
#include <algorithm>

using namespace std;
using namespace boost;

using boost::numeric::ublas::project;
using boost::numeric::ublas::matrix;

// FROM runModel_v2.cpp
/////////////////////////////////////////////////////////////////////
// expect a csv file of data
void LoadData(const string& file, matrix<double>& M) {
  ifstream in(file.c_str());
  if (!in.is_open()) return;
  typedef tokenizer< char_separator<char> > Tokenizer;
  char_separator<char> sep(",");

  string line;
  int nrows = 0; 
  int ncols = 0;
  vector<string> vec;

  // get the size first
  while (std::getline(in,line)) {
    Tokenizer tok(line, sep);
    vec.assign(tok.begin(), tok.end());
    ncols = vec.end() - vec.begin();
    nrows++;
  }
  cout << "num rows = "<< nrows << "  num cols = " << ncols << endl;

  // create a matrix to hold data
  matrix<double> Data(nrows, ncols);
  
  // make second pass 
  in.clear();
  in.seekg(0);
  int r = 0;
  while (std::getline(in,line)) {
    Tokenizer tok(line, sep);
    vec.assign(tok.begin(), tok.end());
    unsigned int i = 0;
    for(i=0; i < vec.size() ; i++) {
      Data(r, i) = ::strtod(vec[i].c_str(), 0);
    }
    r++;
  }
  M = Data;
}

bool is_almost(double val1, double val2, double precision) {
  return abs(val1-val2) < precision;
}

// linspace(a, b, n)
//
//      Return a vector of n equidistant points between a and b,
//      inclusive, for n >= 2.  That is,
//
//              [a, a + s, a + 2s, a + 3s, ..., a + (n - 2)s, b],
//
//      where s = (b - a)/(n - 1) is the distance between consecutive
//      points.  If v is the result, then v[0] = a and v[n - 1] = b.
vector<double> linspace(double a, double b, size_t n) {
  assert(2 <= n);
  assert(a <= b);

  vector<double> v(n);
  const double s = (b - a)/(n - 1);

  v[0] = a;
  for (size_t i = 1; i < (n - 1); i++)
    v[i] = a + i*s;
  v[n - 1] = b;

  return v;
}

// log_linspace(a, b, n)
//
//      Return a vector of n `equiratioed' points between a and b,
//      inclusive, for n >= 2.  That is,
//
//              [a, a s, a s^2, a s^3, ..., a s^(n - 2), b],
//
//      where s = (b/a)^1/(n - 1) is the ratio of consecutive points.
//      If v is the result, then v[0] = a and v[n - 1] = b, except in
//      the case described below.
//
//      If a or b is too close to zero to be represented normally, it
//      is first rounded up to the smallest positive normal value.
//      Analytically, zero makes no sense for either a or b, but
//      values sufficiently close to zero will have been rounded to
//      zero beforehand.
//
//      We ignore subnormals because starting your computation with
//      them is unlikely to help you but quite likely to slow you
//      down.
vector<double> log_linspace(double a, double b, size_t n) {
  assert(2 <= n);
  assert(a <= b);

  a = std::max(a, std::numeric_limits<double>::min());
  b = std::max(b, std::numeric_limits<double>::min());

  vector<double> v(n);
  const double log_a = log(a);
  const double log_r = (log(b) - log_a)/(n - 1);

  v[0] = a;
  for (size_t i = 1; i < (n - 1); i++)
    v[i] = std::max(a, std::min(b, exp(log_a + i*log_r)));
  v[n - 1] = b;

  return v;
}

vector<double> std_vector_add(const vector<double>& vec1,
                              const vector<double>& vec2) {
  assert(vec1.size()==vec2.size());
{'fraction_non_alphanumeric': 0.17324870709920076, 'fraction_numerical': 0.012929007992477669, 'mean_word_length': 2.6085665818490247, 'pattern_counts': {'":': 0, '<': 96, '<?xml version=': 0, '>': 64, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '78506', 'n_tokens_mistral': 3063, 'n_tokens_neox': 2969, 'n_words': 1004}